feat(render): ambient occlusion with bent normals darkens only the light of the surroundings instead of the whole image, so the sun and headlights keep their full strength; the occlusion and the bent normal of the previous frame are reprojected into the lighting of the scene, with the motion of the camera and of every vehicle and cab taken into account and with depth checks against disocclusion, the ambient light follows how much of the sky the bent normal sees, and reflections of the environment are occluded where the cone of visible directions around the bent normal cuts off the reflected lobe, only where the occlusion is clear enough that open flat panels and windows keep their grazing reflections

shaders/gtao_common.glsl

@@ -0,0 +1,17 @@
+vec2 octahedral_encode(vec3 n)
+{
+	n /= abs(n.x) + abs(n.y) + abs(n.z);
+	vec2 o = n.xy;
+	if (n.z < 0.0)
+		o = (1.0 - abs(n.yx)) * vec2(n.x >= 0.0 ? 1.0 : -1.0, n.y >= 0.0 ? 1.0 : -1.0);
+	return o;
+}
+
+vec3 octahedral_decode(vec2 o)
+{
+	vec3 n = vec3(o, 1.0 - abs(o.x) - abs(o.y));
+	float t = clamp(-n.z, 0.0, 1.0);
+	n.x += (n.x >= 0.0 ? -t : t);
+	n.y += (n.y >= 0.0 ? -t : t);
+	return normalize(n);
+}

shaders/light_common.glsl

@@ -4,6 +4,68 @@ uniform sampler2DArrayShadow shadowmap;
 #endif
 uniform sampler2D headlightmap;
 
+#if GTAO_ENABLED
+uniform sampler2D gtaomap;
+
+#include <gtao_common.glsl>
+
+vec4 gtao_sample()
+{
+	vec4 none = vec4(1.0, 0.0, 0.0, 0.0);
+	if (gtao_params.x < 0.5)
+		return none;
+	vec4 clip = gtao_reprojection * vec4(f_pos.xyz, 1.0);
+	if (clip.w <= 0.0)
+		return none;
+	vec2 uv = clip.xy / clip.w * 0.5 + 0.5;
+	if (any(lessThan(uv, vec2(0.0))) || any(greaterThan(uv, vec2(1.0))))
+		return none;
+	ivec2 size = textureSize(gtaomap, 0);
+	vec2 position = uv * vec2(size) - 0.5;
+	ivec2 base = ivec2(floor(position));
+	vec2 f = position - vec2(base);
+	float visibility = 0.0;
+	vec3 bent = vec3(0.0);
+	float weights = 0.0;
+	for (int i = 0; i < 4; ++i)
+	{
+		ivec2 offset = ivec2(i & 1, i >> 1);
+		vec4 tap = texelFetch(gtaomap, clamp(base + offset, ivec2(0), size - 1), 0);
+		float difference = abs(tap.g - clip.w) / (clip.w * 0.08 + 0.05);
+		float weight = (offset.x == 1 ? f.x : 1.0 - f.x) * (offset.y == 1 ? f.y : 1.0 - f.y) * max(1.0 - difference, 0.0);
+		visibility += tap.r * weight;
+		bent += octahedral_decode(tap.ba) * weight;
+		weights += weight;
+	}
+	if (weights < 1e-3 || dot(bent, bent) < 1e-8)
+		return none;
+	return vec4(clamp(visibility / weights, 0.0, 1.0), normalize(bent));
+}
+
+float gtao_cone_overlap(float cos1, float cos2, float cosd)
+{
+	float r1 = acos(cos1);
+	float r2 = acos(cos2);
+	float d = acos(clamp(cosd, -1.0, 1.0));
+	float smaller = 1.0 - max(cos1, cos2);
+	if (min(r1, r2) <= max(r1, r2) - d)
+		return smaller;
+	if (r1 + r2 <= d)
+		return 0.0;
+	float delta = abs(r1 - r2);
+	return smaller * smoothstep(0.0, 1.0, 1.0 - clamp((d - delta) / max(r1 + r2 - delta, 1e-4), 0.0, 1.0));
+}
+
+float gtao_specular(vec4 gtao, vec3 normal, vec3 reflected, float roughness)
+{
+	vec3 bent = (dot(gtao.yzw, gtao.yzw) > 0.5 ? gtao.yzw : normal);
+	float coslobe = clamp(exp2(-3.32193 * roughness * roughness), 0.0, 0.999);
+	float occluded = gtao_cone_overlap(sqrt(clamp(1.0 - gtao.x, 0.0, 1.0)), coslobe, dot(bent, reflected));
+	float open = gtao_cone_overlap(0.0, coslobe, dot(normal, reflected));
+	return mix(1.0, clamp(occluded / max(open, 1e-5), 0.0, 1.0), smoothstep(0.9, 0.5, gtao.x));
+}
+#endif
+
 #include <envmapping.glsl>
 #include <conversion.glsl>
 
@@ -298,6 +360,13 @@ vec3 apply_lights(vec3 fragcolor, vec3 fragnormal, vec3 texturecolor, float refl
     fragcolor *= 1.0 - parallax_occlusion;
 #endif
 
+#if GTAO_ENABLED
+    vec4 gtao = gtao_sample();
+    vec3 gtao_normal = mat3(inv_view) * fragnormal;
+    vec3 gtao_bent = (dot(gtao.yzw, gtao.yzw) > 0.5 ? gtao.yzw : gtao_normal);
+    fragcolor *= pow(gtao.x, gtao_params.y) * min((0.75 + 0.25 * gtao_bent.y) / (0.75 + 0.25 * gtao_normal.y), 1.0);
+#endif
+
     vec3 emissioncolor = vec3(emission);
 
     vec3 view_dir = normalize(-f_pos.xyz);
@@ -314,6 +383,9 @@ vec3 apply_lights(vec3 fragcolor, vec3 fragnormal, vec3 texturecolor, float refl
     // mirroring the sky. `env_spec` is the colour to multiply the cubemap by.
     vec3 env_spec = EnvBRDFApprox(F0, env_roughness, NdotV);
     float env_spec_w = max(env_spec.r, max(env_spec.g, env_spec.b));
+#if GTAO_ENABLED
+    env_spec *= gtao_specular(gtao, gtao_normal, normalize(mat3(inv_view) * reflect(f_pos.xyz, fragnormal)), env_roughness);
+#endif
 
     // Tint texture toward fully-saturated under strong env, weighted by the
     // env BRDF (so a rough/matte surface no longer gets washed toward env hue)

shaders/postfx_gtao.frag

@@ -7,6 +7,8 @@ uniform sampler2D depth_tex;
 
 #include <common>
 
+#include <gtao_common.glsl>
+
 const float PI = 3.14159265;
 const float HALF_PI = 1.57079633;
 
@@ -28,7 +30,7 @@ void main()
 	vec2 size = vec2(textureSize(depth_tex, 0));
 	ivec2 center = ivec2(f_coords * size);
 	if (texelFetch(depth_tex, center, 0).r <= 0.0) {
-		out_color = vec4(1.0);
+		out_color = vec4(1.0, 60000.0, 0.0, 0.0);
 		return;
 	}
 
@@ -45,7 +47,7 @@ void main()
 	float radius = param[0].x;
 	float pixels = radius * projection[1][1] * 0.5 * size.y / -P.z;
 	if (pixels < 2.0) {
-		out_color = vec4(1.0);
+		out_color = vec4(1.0, -P.z, octahedral_encode(mat3(inv_view) * N));
 		return;
 	}
 	pixels = min(pixels, size.y * 0.25);
@@ -59,6 +61,7 @@ void main()
 	float jitter = fract(noise * 1.61803399);
 
 	float visibility = 0.0;
+	vec3 bent = vec3(0.0);
 	for (int d = 0; d < directions; ++d)
 	{
 		float phi = (float(d) + noise) * PI / float(directions);
@@ -95,8 +98,12 @@ void main()
 		float arc0 = (cosn + 2.0 * h0 * sin(n) - cos(2.0 * h0 - n)) * 0.25;
 		float arc1 = (cosn + 2.0 * h1 * sin(n) - cos(2.0 * h1 - n)) * 0.25;
 		visibility += projectedlength * (arc0 + arc1);
+		float middle = 0.5 * (h0 + h1);
+		bent += (V * cos(middle) + normalize(ortho) * sin(middle)) * projectedlength * (arc0 + arc1);
 	}
 	visibility = clamp(visibility / float(directions), 0.0, 1.0);
 	float fade = smoothstep(param[1].y * 0.75, param[1].y, -P.z);
-	out_color = vec4(mix(visibility, 1.0, fade));
+	bent = (dot(bent, bent) > 1e-8 ? normalize(bent) : N);
+	bent = normalize(mix(bent, N, fade));
+	out_color = vec4(mix(visibility, 1.0, fade), -P.z, octahedral_encode(mat3(inv_view) * bent));
 }

shaders/postfx_gtao_apply.frag

@@ -1,13 +0,0 @@
-in vec2 f_coords;
-
-layout(location = 0) out vec4 out_color;
-
-#texture (ao_tex, 0, R)
-uniform sampler2D ao_tex;
-
-#include <common>
-
-void main()
-{
-	out_color = vec4(vec3(pow(clamp(texture(ao_tex, f_coords).r, 0.0, 1.0), param[0].y)), 1.0);
-}

shaders/postfx_gtao_blur.frag

@@ -2,44 +2,37 @@ in vec2 f_coords;
 
 layout(location = 0) out vec4 out_color;
 
-#texture (ao_tex, 0, R)
+#texture (ao_tex, 0, RGBA)
 uniform sampler2D ao_tex;
 
-#texture (depth_tex, 1, R)
-uniform sampler2D depth_tex;
-
 #include <common>
-
-float view_distance(vec2 uv)
-{
-	ivec2 size = textureSize(depth_tex, 0);
-	float depth = texelFetch(depth_tex, clamp(ivec2(uv * vec2(size)), ivec2(0), size - 1), 0).r;
-	if (depth <= 0.0)
-		return 1e6;
-	float ndcz = (param[1].x > 0.5 ? depth : depth * 2.0 - 1.0);
-	return projection[3][2] / (ndcz + projection[2][2]);
-}
+#include <gtao_common.glsl>
 
 void main()
 {
-	vec2 texel = 1.0 / vec2(textureSize(ao_tex, 0));
-	float center = view_distance(f_coords);
-	if (center >= 1e6) {
-		out_color = vec4(1.0);
+	ivec2 size = textureSize(ao_tex, 0);
+	ivec2 center = ivec2(f_coords * vec2(size));
+	vec4 centersample = texelFetch(ao_tex, center, 0);
+	float centerdistance = centersample.g;
+	if (centerdistance >= 60000.0) {
+		out_color = centersample;
 		return;
 	}
-	float sum = 0.0;
+	float visibility = 0.0;
+	vec3 bent = vec3(0.0);
 	float weights = 0.0;
 	for (int y = -2; y <= 2; ++y)
 	{
 		for (int x = -2; x <= 2; ++x)
 		{
-			vec2 uv = f_coords + vec2(float(x), float(y)) * texel;
-			float difference = (view_distance(uv) - center) / (center * 0.02 + 0.01);
+			vec4 tap = texelFetch(ao_tex, clamp(center + ivec2(x, y), ivec2(0), size - 1), 0);
+			float difference = (tap.g - centerdistance) / (centerdistance * 0.02 + 0.01);
 			float weight = exp(-float(x * x + y * y) / 4.5 - difference * difference);
-			sum += texture(ao_tex, uv).r * weight;
+			visibility += tap.r * weight;
+			bent += octahedral_decode(tap.ba) * weight;
 			weights += weight;
 		}
 	}
-	out_color = vec4(sum / max(weights, 1e-4));
+	bent = (dot(bent, bent) > 1e-8 ? normalize(bent) : octahedral_decode(centersample.ba));
+	out_color = vec4(visibility / max(weights, 1e-4), centerdistance, octahedral_encode(bent));
 }