Fix T87541 EEVEE: AO causes black outline around objects and NaN pixels

It seems the pow result is unstable on some implementations.

Also avoid undefined behavior by clamping aoFactor to strict positive values.

3 files changed, 5 insertions, 2 deletions

diff --git a/source/blender/draw/engines/eevee/eevee_occlusion.c b/source/blender/draw/engines/eevee/eevee_occlusion.c
index 3f198063c47..19f34fa6108 100644
--- a/source/blender/draw/engines/eevee/eevee_occlusion.c
+++ b/source/blender/draw/engines/eevee/eevee_occlusion.c
@@ -61,7 +61,7 @@ int EEVEE_occlusion_init(EEVEE_ViewLayerData *sldata, EEVEE_Data *vedata)
     const int fs_size[2] = {(int)viewport_size[0], (int)viewport_size[1]};
 
     common_data->ao_dist = scene_eval->eevee.gtao_distance;
-    common_data->ao_factor = scene_eval->eevee.gtao_factor;
+    common_data->ao_factor = max_ff(1e-4f, scene_eval->eevee.gtao_factor);
     common_data->ao_quality = scene_eval->eevee.gtao_quality;
 
     if (scene_eval->eevee.flag & SCE_EEVEE_GTAO_ENABLED) {
diff --git a/source/blender/draw/engines/eevee/shaders/ambient_occlusion_lib.glsl b/source/blender/draw/engines/eevee/shaders/ambient_occlusion_lib.glsl
index 69b9ecaf77d..bf5b63cba65 100644
--- a/source/blender/draw/engines/eevee/shaders/ambient_occlusion_lib.glsl
+++ b/source/blender/draw/engines/eevee/shaders/ambient_occlusion_lib.glsl
@@ -386,7 +386,8 @@ float specular_occlusion(
   float specular_solid_angle = spherical_cap_intersection(M_PI_2, spec_angle, cone_nor_dist);
   float specular_occlusion = isect_solid_angle / specular_solid_angle;
   /* Mix because it is unstable in unoccluded areas. */
-  visibility = mix(specular_occlusion, 1.0, pow(visibility, 8.0));
+  float tmp = saturate(pow8(visibility));
+  visibility = mix(specular_occlusion, 1.0, tmp);
 
   /* Scale by user factor */
   visibility = pow(saturate(visibility), aoFactor);
diff --git a/source/blender/draw/intern/shaders/common_math_lib.glsl b/source/blender/draw/intern/shaders/common_math_lib.glsl
index 0344b977139..378f27056fa 100644
--- a/source/blender/draw/intern/shaders/common_math_lib.glsl
+++ b/source/blender/draw/intern/shaders/common_math_lib.glsl
@@ -91,6 +91,8 @@ vec2 sqr(vec2 a) { return a * a; }
 vec3 sqr(vec3 a) { return a * a; }
 vec4 sqr(vec4 a) { return a * a; }
 
+float pow8(float x) { return sqr(sqr(sqr(x))); }
+
 float len_squared(vec3 a) { return dot(a, a); }
 float len_squared(vec2 a) { return dot(a, a); }