Fix for [#24293] Shadow pass is wrong

* The problem is that shadow pass is derived from the diffuse pass as
  shad = shad'/diff, where shad' = shad*diff. In cases where diff is
  0 and the division can't be done shad is left as shad' (=0).
* This all works just fine until the diffuse color is 0 on just one
  channel (no red in material color for example). In this case the shadow
  pass is left as 0 too regardless of the existence of an actual shadow,
  so the end result is a colored shadow!
* The only real solution is to use the original shadow intensity to
  determine if there actually is a shadow or not. This is now stored in
  shr->shad[3] from the lamp shadow calculation.

Note: The best solution would probably be to calculate the shadow pass on
it's own and not to derive it from the diffuse pass, but I didn't dare to
start messing up the shading code totally.

2 files changed, 10 insertions, 2 deletions

diff --git a/source/blender/render/extern/include/RE_shader_ext.h b/source/blender/render/extern/include/RE_shader_ext.h
index 6cab4a7ce03..71ce269fce6 100644
--- a/source/blender/render/extern/include/RE_shader_ext.h
+++ b/source/blender/render/extern/include/RE_shader_ext.h
@@ -52,7 +52,7 @@ typedef struct ShadeResult
 	float emit[3];
 	float diff[3];		/* no ramps, shadow, etc */
 	float spec[3];
-	float shad[3];
+	float shad[4];		/* shad[3] is shadow intensity */
 	float ao[3];
 	float env[3];
 	float indirect[3];
diff --git a/source/blender/render/intern/source/shadeoutput.c b/source/blender/render/intern/source/shadeoutput.c
index 4142522e003..bced6dc7544 100644
--- a/source/blender/render/intern/source/shadeoutput.c
+++ b/source/blender/render/intern/source/shadeoutput.c
@@ -1396,6 +1396,7 @@ static void shade_one_light(LampRen *lar, ShadeInput *shi, ShadeResult *shr, int
 					}
 					
 					i*= shadfac[3];
+					shr->shad[3] = shadfac[3]; /* store this for possible check in troublesome cases */
 				}
 			}
 		}
@@ -1730,10 +1731,17 @@ void shade_lamp_loop(ShadeInput *shi, ShadeResult *shr)
 			VECCOPY(shr->combined, shr->diff);
 			
 		/* calculate shadow pass, we use a multiplication mask */
-		if(passflag & SCE_PASS_SHADOW) {
+		/* if diff = 0,0,0 it doesn't matter what the shadow pass is, so leave it as is */
+		if(passflag & SCE_PASS_SHADOW && !(shr->diff[0]==0.0f && shr->diff[1]==0.0f && shr->diff[2]==0.0f)) {
 			if(shr->diff[0]!=0.0f) shr->shad[0]= shr->shad[0]/shr->diff[0];
+			/* can't determine proper shadow from shad/diff (0/0), so use shadow intensity */
+			else if(shr->shad[0]==0.0f) shr->shad[0]= shr->shad[3];
+
 			if(shr->diff[1]!=0.0f) shr->shad[1]= shr->shad[1]/shr->diff[1];
+			else if(shr->shad[1]==0.0f) shr->shad[1]= shr->shad[3];
+
 			if(shr->diff[2]!=0.0f) shr->shad[2]= shr->shad[2]/shr->diff[2];
+			else if(shr->shad[2]==0.0f) shr->shad[2]= shr->shad[3];
 		}
 		
 		/* exposure correction */