Fix T52229: Shadow Catcher artifacts when under transparency

Added some extra tirckery to avoid background being tinted dark with transparent
surface. Maybe a bit hacky, but seems to work fine.

6 files changed, 40 insertions, 23 deletions

diff --git a/intern/cycles/kernel/kernel_accumulate.h b/intern/cycles/kernel/kernel_accumulate.h
index 9ed16aceb55..d139e28b013 100644
--- a/intern/cycles/kernel/kernel_accumulate.h
+++ b/intern/cycles/kernel/kernel_accumulate.h
@@ -21,6 +21,9 @@ CCL_NAMESPACE_BEGIN
  * BSDF evaluation result, split per BSDF type. This is used to accumulate
  * render passes separately. */
 
+ccl_device float3 shader_bsdf_transparency(KernelGlobals *kg,
+                                           const ShaderData *sd);
+
 ccl_device_inline void bsdf_eval_init(BsdfEval *eval, ClosureType type, float3 value, int use_light_pass)
 {
 #ifdef __PASSES__
@@ -223,6 +226,7 @@ ccl_device_inline void path_radiance_init(PathRadiance *L, int use_light_pass)
 	L->shadow_background_color = make_float3(0.0f, 0.0f, 0.0f);
 	L->shadow_radiance_sum = make_float3(0.0f, 0.0f, 0.0f);
 	L->shadow_throughput = 0.0f;
+	L->shadow_transparency = 1.0f;
 #endif
 
 #ifdef __DENOISING_FEATURES__
@@ -398,10 +402,11 @@ ccl_device_inline void path_radiance_accum_total_light(
 #endif
 }
 
-ccl_device_inline void path_radiance_accum_background(PathRadiance *L,
-                                                      ccl_addr_space PathState *state,
-                                                      float3 throughput,
-                                                      float3 value)
+ccl_device_inline void path_radiance_accum_background(
+        PathRadiance *L,
+        ccl_addr_space PathState *state,
+        float3 throughput,
+        float3 value)
 {
 #ifdef __PASSES__
 	if(L->use_light_pass) {
@@ -421,9 +426,7 @@ ccl_device_inline void path_radiance_accum_background(PathRadiance *L,
 #ifdef __SHADOW_TRICKS__
 	if(state->flag & PATH_RAY_STORE_SHADOW_INFO) {
 		L->path_total += throughput * value;
-		if(state->flag & PATH_RAY_SHADOW_CATCHER_ONLY) {
-			L->path_total_shaded += throughput * value;
-		}
+		L->path_total_shaded += throughput * value * L->shadow_transparency;
 	}
 #endif
 
@@ -671,7 +674,7 @@ ccl_device_inline float path_radiance_sum_shadow(const PathRadiance *L)
 	if(path_total != 0.0f) {
 		return path_total_shaded / path_total;
 	}
-	return 1.0f;
+	return L->shadow_transparency;
 }
 
 /* Calculate final light sum and transparency for shadow catcher object. */
diff --git a/intern/cycles/kernel/kernel_path.h b/intern/cycles/kernel/kernel_path.h
index 21564e81b7a..92b31d46697 100644
--- a/intern/cycles/kernel/kernel_path.h
+++ b/intern/cycles/kernel/kernel_path.h
@@ -320,8 +320,12 @@ ccl_device void kernel_path_indirect(KernelGlobals *kg,
 #endif  /* __BRANCHED_PATH__ */
 
 #ifdef __SHADOW_TRICKS__
-		if(!(sd->object_flag & SD_OBJECT_SHADOW_CATCHER)) {
-			state->flag &= ~PATH_RAY_SHADOW_CATCHER_ONLY;
+		if(!(sd->object_flag & SD_OBJECT_SHADOW_CATCHER) &&
+		   (state->flag & PATH_RAY_SHADOW_CATCHER))
+		{
+			/* Only update transparency after shadow catcher bounce. */
+			L->shadow_transparency *=
+				average(shader_bsdf_transparency(kg, sd));
 		}
 #endif  /* __SHADOW_TRICKS__ */
 
@@ -647,7 +651,6 @@ ccl_device_inline float kernel_path_integrate(KernelGlobals *kg,
 		if((sd.object_flag & SD_OBJECT_SHADOW_CATCHER)) {
 			if(state.flag & PATH_RAY_CAMERA) {
 				state.flag |= (PATH_RAY_SHADOW_CATCHER |
-				               PATH_RAY_SHADOW_CATCHER_ONLY |
 				               PATH_RAY_STORE_SHADOW_INFO);
 				if(!kernel_data.background.transparent) {
 					L->shadow_background_color =
@@ -657,8 +660,10 @@ ccl_device_inline float kernel_path_integrate(KernelGlobals *kg,
 				L->shadow_throughput = average(throughput);
 			}
 		}
-		else {
-			state.flag &= ~PATH_RAY_SHADOW_CATCHER_ONLY;
+		else if(state.flag & PATH_RAY_SHADOW_CATCHER) {
+			/* Only update transparency after shadow catcher bounce. */
+			L->shadow_transparency *=
+			        average(shader_bsdf_transparency(kg, &sd));
 		}
 #endif  /* __SHADOW_TRICKS__ */
 
diff --git a/intern/cycles/kernel/kernel_path_branched.h b/intern/cycles/kernel/kernel_path_branched.h
index 4ec37d225f7..cea677fd701 100644
--- a/intern/cycles/kernel/kernel_path_branched.h
+++ b/intern/cycles/kernel/kernel_path_branched.h
@@ -119,6 +119,9 @@ ccl_device_noinline void kernel_branched_path_surface_indirect_light(KernelGloba
 			PathState ps = *state;
 			float3 tp = throughput;
 			Ray bsdf_ray;
+#ifdef __SHADOW_TRICKS__
+			float shadow_transparency = L->shadow_transparency;
+#endif
 
 			if(!kernel_branched_path_surface_bounce(kg,
 			                                        &bsdf_rng,
@@ -149,6 +152,10 @@ ccl_device_noinline void kernel_branched_path_surface_indirect_light(KernelGloba
 			 * for the next samples */
 			path_radiance_sum_indirect(L);
 			path_radiance_reset_indirect(L);
+
+#ifdef __SHADOW_TRICKS__
+			L->shadow_transparency = shadow_transparency;
+#endif
 		}
 	}
 }
@@ -500,7 +507,6 @@ ccl_device float kernel_branched_path_integrate(KernelGlobals *kg,
 #ifdef __SHADOW_TRICKS__
 		if((sd.object_flag & SD_OBJECT_SHADOW_CATCHER)) {
 			state.flag |= (PATH_RAY_SHADOW_CATCHER |
-			               PATH_RAY_SHADOW_CATCHER_ONLY |
 			               PATH_RAY_STORE_SHADOW_INFO);
 			if(!kernel_data.background.transparent) {
 				L->shadow_background_color =
@@ -509,8 +515,10 @@ ccl_device float kernel_branched_path_integrate(KernelGlobals *kg,
 			L->shadow_radiance_sum = path_radiance_clamp_and_sum(kg, L);
 			L->shadow_throughput = average(throughput);
 		}
-		else {
-			state.flag &= ~PATH_RAY_SHADOW_CATCHER_ONLY;
+		else if(state.flag & PATH_RAY_SHADOW_CATCHER) {
+			/* Only update transparency after shadow catcher bounce. */
+			L->shadow_transparency *=
+				average(shader_bsdf_transparency(kg, &sd));
 		}
 #endif  /* __SHADOW_TRICKS__ */
 
diff --git a/intern/cycles/kernel/kernel_shader.h b/intern/cycles/kernel/kernel_shader.h
index 90b936d83c9..f553599a2e4 100644
--- a/intern/cycles/kernel/kernel_shader.h
+++ b/intern/cycles/kernel/kernel_shader.h
@@ -669,7 +669,7 @@ ccl_device void shader_bsdf_blur(KernelGlobals *kg, ShaderData *sd, float roughn
 	}
 }
 
-ccl_device float3 shader_bsdf_transparency(KernelGlobals *kg, ShaderData *sd)
+ccl_device float3 shader_bsdf_transparency(KernelGlobals *kg, const ShaderData *sd)
 {
 	if(sd->flag & SD_HAS_ONLY_VOLUME)
 		return make_float3(1.0f, 1.0f, 1.0f);
@@ -677,7 +677,7 @@ ccl_device float3 shader_bsdf_transparency(KernelGlobals *kg, ShaderData *sd)
 	float3 eval = make_float3(0.0f, 0.0f, 0.0f);
 
 	for(int i = 0; i < sd->num_closure; i++) {
-		ShaderClosure *sc = &sd->closure[i];
+		const ShaderClosure *sc = &sd->closure[i];
 
 		if(sc->type == CLOSURE_BSDF_TRANSPARENT_ID) // todo: make this work for osl
 			eval += sc->weight;
diff --git a/intern/cycles/kernel/kernel_types.h b/intern/cycles/kernel/kernel_types.h
index 90a3c818214..39840d265b2 100644
--- a/intern/cycles/kernel/kernel_types.h
+++ b/intern/cycles/kernel/kernel_types.h
@@ -351,8 +351,7 @@ enum PathRayFlag {
 	PATH_RAY_DIFFUSE_ANCESTOR    = (1 << 16),
 	PATH_RAY_SINGLE_PASS_DONE    = (1 << 17),
 	PATH_RAY_SHADOW_CATCHER      = (1 << 18),
-	PATH_RAY_SHADOW_CATCHER_ONLY = (1 << 19),
-	PATH_RAY_STORE_SHADOW_INFO   = (1 << 20),
+	PATH_RAY_STORE_SHADOW_INFO   = (1 << 19),
 };
 
 /* Closure Label */
@@ -529,6 +528,7 @@ typedef ccl_addr_space struct PathRadiance {
 	 */
 	float3 shadow_radiance_sum;
 	float shadow_throughput;
+	float shadow_transparency;
 #endif
 
 #ifdef __DENOISING_FEATURES__
diff --git a/intern/cycles/kernel/split/kernel_holdout_emission_blurring_pathtermination_ao.h b/intern/cycles/kernel/split/kernel_holdout_emission_blurring_pathtermination_ao.h
index 916b81faf4e..fe761305bfb 100644
--- a/intern/cycles/kernel/split/kernel_holdout_emission_blurring_pathtermination_ao.h
+++ b/intern/cycles/kernel/split/kernel_holdout_emission_blurring_pathtermination_ao.h
@@ -127,7 +127,6 @@ ccl_device void kernel_holdout_emission_blurring_pathtermination_ao(
 			if(state->flag & PATH_RAY_CAMERA) {
 				PathRadiance *L = &kernel_split_state.path_radiance[ray_index];
 				state->flag |= (PATH_RAY_SHADOW_CATCHER |
-				                PATH_RAY_SHADOW_CATCHER_ONLY |
 				                PATH_RAY_STORE_SHADOW_INFO);
 				if(!kernel_data.background.transparent) {
 					ccl_global Ray *ray = &kernel_split_state.ray[ray_index];
@@ -141,8 +140,10 @@ ccl_device void kernel_holdout_emission_blurring_pathtermination_ao(
 				L->shadow_throughput = average(throughput);
 			}
 		}
-		else {
-			state->flag &= ~PATH_RAY_SHADOW_CATCHER_ONLY;
+		else if(state->flag & PATH_RAY_SHADOW_CATCHER) {
+			/* Only update transparency after shadow catcher bounce. */
+			PathRadiance *L = &kernel_split_state.path_radiance[ray_index];
+			L->shadow_transparency *= average(shader_bsdf_transparency(kg, sd));
 		}
 #endif  /* __SHADOW_TRICKS__ */