Let the instances drawn from the GPU hide one another, in two phases

shaders/instance_cull.comp

@@ -1,9 +1,15 @@
 // culls the model instances of the buckets queued in a render pass and assigns them their lod bands, with the tests
 // opengl46_renderer::Render_Instanced() makes on the cpu. one invocation per record of a queued bucket.
-// stage 0 counts the instances which pass, per entry (lod band of a group); after instance_cull_offsets.comp turned the
-// counts into the first slots of the entries, stage 1 writes their modelview matrices there, in no particular order.
+// stage 0 counts the instances which pass the tests, per entry (lod band of a group); after instance_cull_offsets.comp
+// turned the counts into the first slots of the entries, stage 1 writes the modelview matrices of the instances drawn first
+// there, in no particular order, and stage 2 (two phases) those of the instances drawn second after them.
 // with the occlusion test on, an instance whose bounding sphere lies whole behind the depth already drawn in the pass
-// (the farthest depth of the pyramid over the part of the view it takes, see hiz_build.comp) is left out as well
+// (the farthest depth of the pyramid over the part of the view it takes, see hiz_build.comp) is left out as well.
+// in two phases (opengl46_instance_cull::dispatch_second()) stage 1 takes only the instances seen in the previous frame,
+// which are drawn, the pyramid is made again with them in it, and stage 2 tests the others against it; it also keeps for
+// the next frame which instances it found in view. an instance seen before but hidden behind the depth of the first
+// pyramid is hidden behind the second one as well (it only holds nearer depth), so each instance is drawn at most once,
+// and everything left out is hidden behind something drawn
 
 #include <common>
 
@@ -17,7 +23,7 @@ struct instance_record
 	float rangesquaredmin;
 	float rangesquaredmax;
 	uint flags;
-	uint pad;
+	uint visible; // the instance was seen in the previous frame (written by stage 2, 0 when the cpu sends the record)
 	vec4 rotationscale[3];
 };
 
@@ -41,7 +47,7 @@ struct instance_cull_group
 	uint pad;
 };
 
-layout(std430, binding = INSTANCE_RECORD_SSBO_BINDING) readonly buffer instance_record_ssbo
+layout(std430, binding = INSTANCE_RECORD_SSBO_BINDING) buffer instance_record_ssbo
 {
 	instance_record records[];
 };
@@ -82,6 +88,8 @@ layout(location = 10) uniform mat4 hiz_projection;
 layout(location = 14) uniform ivec4 hiz_params;
 // xy: size of the depth the pyramid was made from, pixels
 layout(location = 15) uniform vec4 hiz_screen;
+// x: two phases, stage 1 takes only the instances seen in the previous frame
+layout(location = 16) uniform uvec4 phases;
 
 // depth pyramid, farthest (smallest, reversed) depth per texel
 layout(binding = 0) uniform sampler2D hiz;
@@ -90,6 +98,9 @@ const uint INSTANCE_VISIBLE = 1u;
 const uint INSTANCE_INSTANCEABLE = 2u;
 // smallest part of the view an instance has to take
 const float MIN_SCREEN_SIZE = 0.003;
+// header of the counters, see gl::INSTANCE_COUNTER_HEADER
+const uint COUNTER_OCCLUDED = 3u;
+const uint COUNTER_FIRST = 4u;
 
 // the job which holds the given record: the last one starting at it or before it
 uint find_job(uint candidate)
@@ -151,43 +162,33 @@ bool occluded(vec3 center, float radius)
 	return nearest < farthest;
 }
 
-void main()
+// the tests of the cpu (Render_Instanced()), all but the occlusion one: the entry of the instance, or false when it's left out.
+// Location: relative to the camera of the pass
+bool passes(instance_cull_job job, instance_record record, out vec3 location, out uint entry)
 {
-	// the dispatch can be two dimensional when the records don't fit in the work group count limit of one
-	uint candidate = (gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x) * INSTANCE_CULL_GROUP_SIZE + gl_LocalInvocationIndex;
-	if (candidate >= counts.y)
-		return;
-
-	instance_cull_job job = jobs[find_job(candidate)];
-	instance_record record = records[job.first + (candidate - job.candidatestart)];
+	location = job.origin + record.position;
+	entry = 0u;
 	// instances hidden, or drawn on their own after getting animations or skins from events
 	if ((record.flags & (INSTANCE_VISIBLE | INSTANCE_INSTANCEABLE)) != (INSTANCE_VISIBLE | INSTANCE_INSTANCEABLE))
-		return;
+		return false;
 
-	vec3 location = job.origin + record.position;
 	for (int plane = 0; plane < 6; ++plane)
 	{
 		if (dot(planes[plane].xyz, location) + planes[plane].w <= -record.radius)
-			return;
+			return false;
 	}
 
 	vec3 offset = job.distanceorigin + record.position;
 	float distancesquared = dot(offset, offset) * distance_params.x;
 	if (distancesquared > distance_params.z)
-		return;
+		return false;
 	distancesquared += distance_params.y;
 	if ((distancesquared < record.rangesquaredmin) || (distancesquared >= record.rangesquaredmax))
-		return;
+		return false;
 	if (distancesquared > distance_params.w)
-		return;
+		return false;
 	if (record.radius * record.radius * size_params.x / distancesquared < MIN_SCREEN_SIZE * MIN_SCREEN_SIZE)
-		return;
-	if ((hiz_params.x != 0) && occluded((view * vec4(location, 1.0)).xyz, record.radius))
-	{
-		if (counts.w == 0u)
-			atomicAdd(counters[3], 1u);
-		return;
-	}
+		return false;
 
 	// band: the number of lod bounds the distance reached, like upper_bound over the sorted bounds
 	instance_cull_group group = groups[job.group];
@@ -197,15 +198,65 @@ void main()
 		if (lodbounds[group.lodfirst + bound] <= distancesquared)
 			band = bound + 1u;
 	}
-	uint entry = group.entryfirst + band;
+	entry = group.entryfirst + band;
+	return true;
+}
+
+void write(uint entry, instance_record record, vec3 location)
+{
+	uint slot = atomicAdd(counters[INSTANCE_COUNTER_HEADER + counts.z + entry], 1u);
+	modelviews[slot] = view * mat4(record.rotationscale[0], record.rotationscale[1], record.rotationscale[2], vec4(location, 1.0));
+}
+
+void main()
+{
+	// the dispatch can be two dimensional when the records don't fit in the work group count limit of one
+	uint candidate = (gl_WorkGroupID.y * gl_NumWorkGroups.x + gl_WorkGroupID.x) * INSTANCE_CULL_GROUP_SIZE + gl_LocalInvocationIndex;
+	if (candidate >= counts.y)
+		return;
+
+	instance_cull_job job = jobs[find_job(candidate)];
+	uint index = job.first + (candidate - job.candidatestart);
+	instance_record record = records[index];
+	vec3 location;
+	uint entry;
+	bool passed = passes(job, record, location, entry);
 
 	if (counts.w == 0u)
 	{
-		atomicAdd(counters[INSTANCE_COUNTER_HEADER + entry], 1u);
+		// room in the output for every instance which passes, whatever the occlusion test says
+		if (passed)
+			atomicAdd(counters[INSTANCE_COUNTER_HEADER + entry], 1u);
+		return;
 	}
-	else
+
+	bool seen = (record.visible != 0u);
+	if (counts.w == 1u)
 	{
-		uint slot = atomicAdd(counters[INSTANCE_COUNTER_HEADER + counts.z + entry], 1u);
-		modelviews[slot] = view * mat4(record.rotationscale[0], record.rotationscale[1], record.rotationscale[2], vec4(location, 1.0));
+		// the ones seen in the previous frame first, the others wait for stage 2
+		if ((false == passed) || ((phases.x != 0u) && (false == seen)))
+			return;
+		if ((hiz_params.x != 0) && occluded((view * vec4(location, 1.0)).xyz, record.radius))
+		{
+			atomicAdd(counters[COUNTER_OCCLUDED], 1u);
+			return;
+		}
+		write(entry, record, location);
+		if (phases.x != 0u)
+			atomicAdd(counters[COUNTER_FIRST], 1u);
+		return;
 	}
+
+	// stage 2, against the pyramid with the instances drawn first in it
+	bool inview = passed && (false == occluded((view * vec4(location, 1.0)).xyz, record.radius));
+	if (passed && (false == seen))
+	{
+		if (inview)
+			write(entry, record, location);
+		else
+			atomicAdd(counters[COUNTER_OCCLUDED], 1u);
+	}
+	uint visible = (inview ? 1u : 0u);
+	if (visible != record.visible)
+		records[index].visible = visible;
 }