Draw instances from the GPU culling with indirect draws, occlusion culling and a depth prepass

shaders/depthprepass.frag

@@ -0,0 +1,7 @@
+// depth prepass of materials which never discard a fragment (no alpha test)
+
+in vec2 f_coord;
+
+void main()
+{
+}

shaders/depthprepass.vert

@@ -0,0 +1,20 @@
+// depth prepass of the instances drawn from the gpu culling (opengl46_renderer::Render_Instances_Indirect()).
+// the position is worked out the way vertex.vert of the materials does it, and declared invariant in both, so the colour
+// pass which follows gets the same depth for the same surface
+
+layout(location = 0) in vec3 v_vert;
+layout(location = 2) in vec2 v_coord;
+
+#include <common>
+
+invariant gl_Position;
+
+out vec2 f_coord;
+
+void main()
+{
+	mat4 mv = effective_modelview();
+	vec4 position = mv * vec4(v_vert, 1.0);
+	gl_Position = projection * position;
+	f_coord = v_coord;
+}

shaders/depthprepass_alpha.frag

@@ -0,0 +1,16 @@
+// depth prepass of materials whose colour shader tests the alpha of the diffuse texture against the opacity of the
+// material (mat_default.frag and the like). the test here is stricter by one step of 8-bit alpha: a fragment which writes
+// depth here is never discarded by the colour pass, which would leave a hole with nothing drawn behind it
+
+in vec2 f_coord;
+
+#include <common>
+
+#texture (diffuse, 0, sRGB_A)
+uniform sampler2D diffuse;
+
+void main()
+{
+	if (texture(diffuse, f_coord).a < opacity + 1.0 / 255.0)
+		discard;
+}

shaders/hiz_build.comp

@@ -0,0 +1,39 @@
+// one level of the depth pyramid of the occlusion test of the instances (opengl46_hiz): each texel holds the farthest
+// depth of the texels of the level below it which it covers. the depth is reversed (1 near, 0 far), so the farthest is
+// the smallest. the first level is made from the resolved depth of the scene, the others from the level below.
+// a level halves the one below, rounding down; when that one has an odd size, the last row and column of the new level
+// take its last row and column as well, so that nothing is left out
+
+#include <common>
+
+layout(local_size_x = HIZ_GROUP_SIZE, local_size_y = HIZ_GROUP_SIZE) in;
+
+layout(binding = 0) uniform sampler2D source;
+layout(r32f, binding = 0) writeonly uniform image2D target;
+
+// xy: size of the source level, zw: size of the target level
+layout(location = 0) uniform ivec4 sizes;
+// x: level of the source read
+layout(location = 1) uniform ivec4 params;
+
+void main()
+{
+	ivec2 texel = ivec2(gl_GlobalInvocationID.xy);
+	if (any(greaterThanEqual(texel, sizes.zw)))
+		return;
+
+	ivec2 first = min(2 * texel, sizes.xy - 1);
+	ivec2 last = min(2 * texel + 1, sizes.xy - 1);
+	if (texel.x == sizes.z - 1)
+		last.x = sizes.x - 1;
+	if (texel.y == sizes.w - 1)
+		last.y = sizes.y - 1;
+
+	float farthest = 1.0;
+	for (int y = first.y; y <= last.y; ++y)
+	{
+		for (int x = first.x; x <= last.x; ++x)
+			farthest = min(farthest, texelFetch(source, ivec2(x, y), params.x).r);
+	}
+	imageStore(target, texel, vec4(farthest));
+}

shaders/instance_cull.comp

@@ -1,7 +1,9 @@
 // culls the model instances of the buckets queued in a render pass and assigns them their lod bands, with the tests
 // opengl46_renderer::Render_Instanced() makes on the cpu. one invocation per record of a queued bucket.
 // stage 0 counts the instances which pass, per entry (lod band of a group); after instance_cull_offsets.comp turned the
-// counts into the first slots of the entries, stage 1 writes their modelview matrices there, in no particular order
+// counts into the first slots of the entries, stage 1 writes their modelview matrices there, in no particular order.
+// with the occlusion test on, an instance whose bounding sphere lies whole behind the depth already drawn in the pass
+// (the farthest depth of the pyramid over the part of the view it takes, see hiz_build.comp) is left out as well
 
 #include <common>
 
@@ -75,6 +77,14 @@ layout(location = 7) uniform vec4 distance_params;
 layout(location = 8) uniform vec4 size_params;
 // x: jobs, y: records in all jobs, z: entries, w: stage
 layout(location = 9) uniform uvec4 counts;
+// occlusion test: projection of the pass, x: on, y: levels of the pyramid, zw: size of its first level
+layout(location = 10) uniform mat4 hiz_projection;
+layout(location = 14) uniform ivec4 hiz_params;
+// xy: size of the depth the pyramid was made from, pixels
+layout(location = 15) uniform vec4 hiz_screen;
+
+// depth pyramid, farthest (smallest, reversed) depth per texel
+layout(binding = 0) uniform sampler2D hiz;
 
 const uint INSTANCE_VISIBLE = 1u;
 const uint INSTANCE_INSTANCEABLE = 2u;
@@ -97,6 +107,50 @@ uint find_job(uint candidate)
 	return low;
 }
 
+// whether a sphere (view space) lies whole behind the depth in the pyramid over the rectangle of the view it takes
+bool occluded(vec3 center, float radius)
+{
+	vec2 low = vec2(1.0);
+	vec2 high = vec2(-1.0);
+	float nearest = 0.0;
+	for (int corner = 0; corner < 8; ++corner)
+	{
+		vec3 offset = vec3(((corner & 1) != 0) ? radius : -radius, ((corner & 2) != 0) ? radius : -radius, ((corner & 4) != 0) ? radius : -radius);
+		vec4 clip = hiz_projection * vec4(center + offset, 1.0);
+		// reaches the plane of the camera: no rectangle to test
+		if (clip.w <= 1e-3)
+			return false;
+		vec3 ndc = clip.xyz / clip.w;
+		low = min(low, ndc.xy);
+		high = max(high, ndc.xy);
+		nearest = max(nearest, ndc.z);
+	}
+	if (any(greaterThan(low, vec2(1.0))) || any(lessThan(high, vec2(-1.0))) || (nearest >= 1.0))
+		return false;
+
+	// pixels of the rectangle, then texels of the first level: each covers 2x2 pixels
+	ivec2 size = hiz_params.zw;
+	ivec2 pixelfirst = ivec2(clamp((low * 0.5 + 0.5) * hiz_screen.xy, vec2(0.0), hiz_screen.xy - 1.0));
+	ivec2 pixellast = ivec2(clamp((high * 0.5 + 0.5) * hiz_screen.xy, vec2(0.0), hiz_screen.xy - 1.0));
+	ivec2 first = min(pixelfirst / 2, size - 1);
+	ivec2 last = min(pixellast / 2, size - 1);
+	// the level where the rectangle takes at most 2x2 texels
+	int level = 0;
+	while ((level < hiz_params.y - 1) && any(greaterThan((last >> level) - (first >> level), ivec2(1))))
+		++level;
+	ivec2 levelsize = max(size >> level, ivec2(1));
+	first = min(first >> level, levelsize - 1);
+	last = min(last >> level, levelsize - 1);
+
+	float farthest = 1.0;
+	for (int y = first.y; y <= last.y; ++y)
+	{
+		for (int x = first.x; x <= last.x; ++x)
+			farthest = min(farthest, texelFetch(hiz, ivec2(x, y), level).r);
+	}
+	return nearest < farthest;
+}
+
 void main()
 {
 	// the dispatch can be two dimensional when the records don't fit in the work group count limit of one
@@ -128,6 +182,12 @@ void main()
 		return;
 	if (record.radius * record.radius * size_params.x / distancesquared < MIN_SCREEN_SIZE * MIN_SCREEN_SIZE)
 		return;
+	if ((hiz_params.x != 0) && occluded((view * vec4(location, 1.0)).xyz, record.radius))
+	{
+		if (counts.w == 0u)
+			atomicAdd(counters[3], 1u);
+		return;
+	}
 
 	// band: the number of lod bounds the distance reached, like upper_bound over the sorted bounds
 	instance_cull_group group = groups[job.group];

shaders/instance_cull_commands.comp

@@ -0,0 +1,39 @@
+// fills in the instances of the indirect draw commands of the entries, after stage 1 of instance_cull.comp wrote the
+// modelview matrices of the instances which passed. one invocation per command: the instance count is the count of its
+// entry, the base instance the first slot of the entry in the output (its cursor, advanced by stage 1, less the count)
+
+#include <common>
+
+layout(local_size_x = INSTANCE_COMMAND_GROUP_SIZE) in;
+
+layout(std430, binding = INSTANCE_COUNTER_SSBO_BINDING) readonly buffer instance_counter_ssbo
+{
+	uint counters[];
+};
+
+// gl::instance_draw_command, eight uints each: count, instance count, first, base vertex (base instance of an array
+// draw), base instance, entry, indexed, unused
+layout(std430, binding = INSTANCE_COMMAND_SSBO_BINDING) buffer instance_command_ssbo
+{
+	uint commands[];
+};
+
+// x: commands, y: entries
+layout(location = 0) uniform uvec4 counts;
+
+void main()
+{
+	uint command = gl_GlobalInvocationID.x;
+	if (command >= counts.x)
+		return;
+
+	uint field = command * 8u;
+	uint entry = commands[field + 5u];
+	uint count = counters[INSTANCE_COUNTER_HEADER + entry];
+	uint base = counters[INSTANCE_COUNTER_HEADER + counts.y + entry] - count;
+	commands[field + 1u] = count;
+	if (commands[field + 6u] != 0u)
+		commands[field + 4u] = base;
+	else
+		commands[field + 3u] = base;
+}

shaders/vertex.vert

@@ -6,6 +6,9 @@ layout(location = 4) in vec4 v_userdata;
 
 #include <common>
 
+// the depth prepass (depthprepass.vert) works the position out the same way, and has to get the same depth
+invariant gl_Position;
+
 out vec3 f_normal;
 flat out vec3 f_normal_raw;
 out vec2 f_coord;