Let the instances drawn from the GPU hide one another, in two phases

shaders/instance_cull.comp

@@ -1,9 +1,15 @@
 // culls the model instances of the buckets queued in a render pass and assigns them their lod bands, with the tests
 // opengl46_renderer::Render_Instanced() makes on the cpu. one invocation per record of a queued bucket.
-// stage 0 counts the instances which pass, per entry (lod band of a group); after instance_cull_offsets.comp turned the
-// counts into the first slots of the entries, stage 1 writes their modelview matrices there, in no particular order.
+// stage 0 counts the instances which pass the tests, per entry (lod band of a group); after instance_cull_offsets.comp
+// turned the counts into the first slots of the entries, stage 1 writes the modelview matrices of the instances drawn first
+// there, in no particular order, and stage 2 (two phases) those of the instances drawn second after them.
 // with the occlusion test on, an instance whose bounding sphere lies whole behind the depth already drawn in the pass
-// (the farthest depth of the pyramid over the part of the view it takes, see hiz_build.comp) is left out as well
+// (the farthest depth of the pyramid over the part of the view it takes, see hiz_build.comp) is left out as well.
+// in two phases (opengl46_instance_cull::dispatch_second()) stage 1 takes only the instances seen in the previous frame,
+// which are drawn, the pyramid is made again with them in it, and stage 2 tests the others against it; it also keeps for
+// the next frame which instances it found in view. an instance seen before but hidden behind the depth of the first
+// pyramid is hidden behind the second one as well (it only holds nearer depth), so each instance is drawn at most once,
+// and everything left out is hidden behind something drawn
 
 #include <common>
 
@@ -17,7 +23,7 @@ struct instance_record
 	float rangesquaredmin;
 	float rangesquaredmax;
 	uint flags;
-	uint pad;
+	uint visible; // the instance was seen in the previous frame (written by stage 2, 0 when the cpu sends the record)
 	vec4 rotationscale[3];
 };
 
@@ -41,7 +47,7 @@ struct instance_cull_group
 	uint pad;
 };
 
-layout(std430, binding = INSTANCE_RECORD_SSBO_BINDING) readonly buffer instance_record_ssbo
+layout(std430, binding = INSTANCE_RECORD_SSBO_BINDING) buffer instance_record_ssbo
 {
 	instance_record records[];
 };
@@ -82,6 +88,8 @@ layout(location = 10) uniform mat4 hiz_projection;
 layout(location = 14) uniform ivec4 hiz_params;
 // xy: size of the depth the pyramid was made from, pixels
 layout(location = 15) uniform vec4 hiz_screen;
+// x: two phases, stage 1 takes only the instances seen in the previous frame
+layout(location = 16) uniform uvec4 phases;
 
 // depth pyramid, farthest (smallest, reversed) depth per texel
 layout(binding = 0) uniform sampler2D hiz;
@@ -90,6 +98,9 @@ const uint INSTANCE_VISIBLE = 1u;
 const uint INSTANCE_INSTANCEABLE = 2u;
 // smallest part of the view an instance has to take
 const float MIN_SCREEN_SIZE = 0.003;
+// header of the counters, see gl::INSTANCE_COUNTER_HEADER
+const uint COUNTER_OCCLUDED = 3u;
+const uint COUNTER_FIRST = 4u;
 
 // the job which holds the given record: the last one starting at it or before it
 uint find_job(uint candidate)
@@ -151,43 +162,33 @@ bool occluded(vec3 center, float radius)
 	return nearest < farthest;
 }
 
-void main()
+// the tests of the cpu (Render_Instanced()), all but the occlusion one: the entry of the instance, or false when it's left out.
+// Location: relative to the camera of the pass
+bool passes(instance_cull_job job, instance_record record, out vec3 location, out uint entry)
 {
-	// the dispatch can be two dimensional when the records don't fit in the work group count limit of one
-	uint candidate = (gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x) * INSTANCE_CULL_GROUP_SIZE + gl_LocalInvocationIndex;
-	if (candidate >= counts.y)
-		return;
-
-	instance_cull_job job = jobs[find_job(candidate)];
-	instance_record record = records[job.first + (candidate - job.candidatestart)];
+	location = job.origin + record.position;
+	entry = 0u;
 	// instances hidden, or drawn on their own after getting animations or skins from events
 	if ((record.flags & (INSTANCE_VISIBLE | INSTANCE_INSTANCEABLE)) != (INSTANCE_VISIBLE | INSTANCE_INSTANCEABLE))
-		return;
+		return false;
 
-	vec3 location = job.origin + record.position;
 	for (int plane = 0; plane < 6; ++plane)
 	{
 		if (dot(planes[plane].xyz, location) + planes[plane].w <= -record.radius)
-			return;
+			return false;
 	}
 
 	vec3 offset = job.distanceorigin + record.position;
 	float distancesquared = dot(offset, offset) * distance_params.x;
 	if (distancesquared > distance_params.z)
-		return;
+		return false;
 	distancesquared += distance_params.y;
 	if ((distancesquared < record.rangesquaredmin) || (distancesquared >= record.rangesquaredmax))
-		return;
+		return false;
 	if (distancesquared > distance_params.w)
-		return;
+		return false;
 	if (record.radius * record.radius * size_params.x / distancesquared < MIN_SCREEN_SIZE * MIN_SCREEN_SIZE)
-		return;
-	if ((hiz_params.x != 0) && occluded((view * vec4(location, 1.0)).xyz, record.radius))
-	{
-		if (counts.w == 0u)
-			atomicAdd(counters[3], 1u);
-		return;
-	}
+		return false;
 
 	// band: the number of lod bounds the distance reached, like upper_bound over the sorted bounds
 	instance_cull_group group = groups[job.group];
@@ -197,15 +198,65 @@ void main()
 		if (lodbounds[group.lodfirst + bound] <= distancesquared)
 			band = bound + 1u;
 	}
-	uint entry = group.entryfirst + band;
+	entry = group.entryfirst + band;
+	return true;
+}
+
+void write(uint entry, instance_record record, vec3 location)
+{
+	uint slot = atomicAdd(counters[INSTANCE_COUNTER_HEADER + counts.z + entry], 1u);
+	modelviews[slot] = view * mat4(record.rotationscale[0], record.rotationscale[1], record.rotationscale[2], vec4(location, 1.0));
+}
+
+void main()
+{
+	// the dispatch can be two dimensional when the records don't fit in the work group count limit of one
+	uint candidate = (gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x) * INSTANCE_CULL_GROUP_SIZE + gl_LocalInvocationIndex;
+	if (candidate >= counts.y)
+		return;
+
+	instance_cull_job job = jobs[find_job(candidate)];
+	uint index = job.first + (candidate - job.candidatestart);
+	instance_record record = records[index];
+	vec3 location;
+	uint entry;
+	bool passed = passes(job, record, location, entry);
 
 	if (counts.w == 0u)
 	{
-		atomicAdd(counters[INSTANCE_COUNTER_HEADER + entry], 1u);
+		// room in the output for every instance which passes, whatever the occlusion test says
+		if (passed)
+			atomicAdd(counters[INSTANCE_COUNTER_HEADER + entry], 1u);
+		return;
 	}
-	else
+
+	bool seen = (record.visible != 0u);
+	if (counts.w == 1u)
 	{
-		uint slot = atomicAdd(counters[INSTANCE_COUNTER_HEADER + counts.z + entry], 1u);
-		modelviews[slot] = view * mat4(record.rotationscale[0], record.rotationscale[1], record.rotationscale[2], vec4(location, 1.0));
+		// the ones seen in the previous frame first, the others wait for stage 2
+		if ((false == passed) || ((phases.x != 0u) && (false == seen)))
+			return;
+		if ((hiz_params.x != 0) && occluded((view * vec4(location, 1.0)).xyz, record.radius))
+		{
+			atomicAdd(counters[COUNTER_OCCLUDED], 1u);
+			return;
+		}
+		write(entry, record, location);
+		if (phases.x != 0u)
+			atomicAdd(counters[COUNTER_FIRST], 1u);
+		return;
 	}
+
+	// stage 2, against the pyramid with the instances drawn first in it
+	bool inview = passed && (false == occluded((view * vec4(location, 1.0)).xyz, record.radius));
+	if (passed && (false == seen))
+	{
+		if (inview)
+			write(entry, record, location);
+		else
+			atomicAdd(counters[COUNTER_OCCLUDED], 1u);
+	}
+	uint visible = (inview ? 1u : 0u);
+	if (visible != record.visible)
+		records[index].visible = visible;
 }

shaders/instance_cull_commands.comp

@@ -1,6 +1,9 @@
-// fills in the instances of the indirect draw commands of the entries, after stage 1 of instance_cull.comp wrote the
-// modelview matrices of the instances which passed. one invocation per command: the instance count is the count of its
-// entry, the base instance the first slot of the entry in the output (its cursor, advanced by stage 1, less the count)
+// fills in the instances of the indirect draw commands of the entries, after instance_cull.comp wrote the modelview
+// matrices of the instances drawn. one invocation per command. the commands are kept three times over: the first copy
+// draws the instances of stage 1, the second all of them, the third those of stage 2 (two phases), each from its first
+// slot in the output on. stage 0 (after stage 1): the instance count of the first copy is how far stage 1 advanced the
+// cursor of its entry, the base instance the first slot of the entry. stage 1 (after stage 2): the other two copies take
+// the command of the first one, the second with the instances of both stages, the third with the ones past the first copy
 
 #include <common>
 
@@ -18,9 +21,18 @@ layout(std430, binding = INSTANCE_COMMAND_SSBO_BINDING) buffer instance_command_
 	uint commands[];
 };
 
-// x: commands, y: entries
+// x: commands (of one copy), y: entries, z: stage
 layout(location = 0) uniform uvec4 counts;
 
+void set_instances(uint field, uint count, uint base)
+{
+	commands[field + 1u] = count;
+	if (commands[field + 6u] != 0u)
+		commands[field + 4u] = base;
+	else
+		commands[field + 3u] = base;
+}
+
 void main()
 {
 	uint command = gl_GlobalInvocationID.x;
@@ -29,11 +41,22 @@ void main()
 
 	uint field = command * 8u;
 	uint entry = commands[field + 5u];
-	uint count = counters[INSTANCE_COUNTER_HEADER + entry];
-	uint base = counters[INSTANCE_COUNTER_HEADER + counts.y + entry] - count;
-	commands[field + 1u] = count;
-	if (commands[field + 6u] != 0u)
-		commands[field + 4u] = base;
-	else
-		commands[field + 3u] = base;
+	uint first = counters[INSTANCE_COUNTER_HEADER + 2u * counts.y + entry];
+	uint count = counters[INSTANCE_COUNTER_HEADER + counts.y + entry] - first;
+	if (counts.z == 0u)
+	{
+		set_instances(field, count, first);
+		return;
+	}
+
+	uint drawnfirst = commands[field + 1u];
+	uint all = (counts.x + command) * 8u;
+	uint second = (2u * counts.x + command) * 8u;
+	for (uint value = 0u; value < 8u; ++value)
+	{
+		commands[all + value] = commands[field + value];
+		commands[second + value] = commands[field + value];
+	}
+	set_instances(all, count, first);
+	set_instances(second, count - drawnfirst, first + drawnfirst);
 }

shaders/instance_cull_offsets.comp

@@ -1,5 +1,6 @@
 // turns the counts of instances of the entries, written by stage 0 of instance_cull.comp, into the first slots of the
-// entries in the output (cursors, which stage 1 advances as it writes), and puts the total in the header.
+// entries in the output (cursors, which stages 1 and 2 advance as they write, and the same first slots kept for
+// instance_cull_commands.comp), and puts the total in the header.
 // a single work group: each invocation sums a run of the entries, then the sums are scanned in shared memory
 
 #include <common>
@@ -46,6 +47,7 @@ void main()
 	for (uint entry = begin; entry < end; ++entry)
 	{
 		counters[INSTANCE_COUNTER_HEADER + entries + entry] = offset;
+		counters[INSTANCE_COUNTER_HEADER + 2u * entries + entry] = offset;
 		offset += counters[INSTANCE_COUNTER_HEADER + entry];
 	}
 	if (invocation == INSTANCE_OFFSETS_GROUP_SIZE - 1u)