Fix #23604: external shadows for volume materials don't work

5 files changed, 27 insertions, 15 deletions

diff --git a/source/blender/render/extern/include/RE_raytrace.h b/source/blender/render/extern/include/RE_raytrace.h
index 2bbcfc0d41c..cf463c0795e 100644
--- a/source/blender/render/extern/include/RE_raytrace.h
+++ b/source/blender/render/extern/include/RE_raytrace.h
@@ -177,6 +177,7 @@ struct Isect
 	int lay;				/* -1 default, set for layer lamps */
 	
 	int skip;				/* RE_SKIP_CULLFACE */
+	int check;
 
 	float col[4];			/* RGBA for shadow_tra */
 
@@ -195,13 +196,15 @@ struct Isect
 #define RE_RAY_SHADOW_TRA 2
 
 /* skip options */
-#define RE_SKIP_CULLFACE		(1 << 0)
-
+#define RE_SKIP_CULLFACE				(1 << 0)
 /* if using this flag then *face should be a pointer to a VlakRen */
 #define RE_SKIP_VLR_NEIGHBOUR			(1 << 1)
-#define RE_SKIP_VLR_RENDER_CHECK		(1 << 2)
-#define RE_SKIP_VLR_NON_SOLID_MATERIAL	(1 << 3)
-#define RE_SKIP_VLR_BAKE_CHECK			(1 << 4)
+
+/* check options */
+#define RE_CHECK_VLR_NONE				0
+#define RE_CHECK_VLR_RENDER				1
+#define RE_CHECK_VLR_NON_SOLID_MATERIAL	2
+#define RE_CHECK_VLR_BAKE				3
 
 /* TODO use: FLT_MAX? */
 #define RE_RAYTRACE_MAXDIST	1e33
diff --git a/source/blender/render/intern/raytrace/rayobject.cpp b/source/blender/render/intern/raytrace/rayobject.cpp
index 119a62ad05a..6a7066eeb9b 100644
--- a/source/blender/render/intern/raytrace/rayobject.cpp
+++ b/source/blender/render/intern/raytrace/rayobject.cpp
@@ -195,19 +195,19 @@ static int intersect_rayface(RayObject *hit_obj, RayFace *face, Isect *is)
 		return 0;
 		
 	/* check if we should intersect this face */
-	if(is->skip & RE_SKIP_VLR_RENDER_CHECK)
+	if(is->check == RE_CHECK_VLR_RENDER)
 	{
 		if(vlr_check_intersect(is, (ObjectInstanceRen*)face->ob, (VlakRen*)face->face ) == 0)
 			return 0;
 	}
-	else if(is->skip & RE_SKIP_VLR_NON_SOLID_MATERIAL)
+	else if(is->check == RE_CHECK_VLR_NON_SOLID_MATERIAL)
 	{
 		if(vlr_check_intersect(is, (ObjectInstanceRen*)face->ob, (VlakRen*)face->face ) == 0)
 			return 0;
 		if(vlr_check_intersect_solid(is, (ObjectInstanceRen*)face->ob, (VlakRen*)face->face) == 0)
 			return 0;
 	}
-	else if(is->skip & RE_SKIP_VLR_BAKE_CHECK) {
+	else if(is->check == RE_CHECK_VLR_BAKE) {
 		if(vlr_check_bake(is, (ObjectInstanceRen*)face->ob, (VlakRen*)face->face ) == 0)
 			return 0;
 	}
diff --git a/source/blender/render/intern/source/rayshade.c b/source/blender/render/intern/source/rayshade.c
index f66d8b9c55e..bde088167c9 100644
--- a/source/blender/render/intern/source/rayshade.c
+++ b/source/blender/render/intern/source/rayshade.c
@@ -707,7 +707,8 @@ static void traceray(ShadeInput *origshi, ShadeResult *origshr, short depth, flo
 	VECCOPY(isec.vec, vec );
 	isec.labda = dist_mir > 0 ? dist_mir : RE_RAYTRACE_MAXDIST;
 	isec.mode= RE_RAY_MIRROR;
-	isec.skip = RE_SKIP_VLR_NEIGHBOUR | RE_SKIP_VLR_RENDER_CHECK;
+	isec.check = RE_CHECK_VLR_RENDER;
+	isec.skip = RE_SKIP_VLR_NEIGHBOUR;
 	isec.hint = 0;
 
 	isec.orig.ob   = obi;
@@ -1850,7 +1851,8 @@ static void ray_ao_qmc(ShadeInput *shi, float *ao, float *env)
 	RE_RC_INIT(isec, *shi);
 	isec.orig.ob   = shi->obi;
 	isec.orig.face = shi->vlr;
-	isec.skip = RE_SKIP_VLR_NEIGHBOUR|RE_SKIP_VLR_NON_SOLID_MATERIAL;
+	isec.check = RE_CHECK_VLR_NON_SOLID_MATERIAL;
+	isec.skip = RE_SKIP_VLR_NEIGHBOUR;
 	isec.hint = 0;
 
 	isec.hit.ob   = 0;
@@ -1989,7 +1991,8 @@ static void ray_ao_spheresamp(ShadeInput *shi, float *ao, float *env)
 	RE_RC_INIT(isec, *shi);
 	isec.orig.ob   = shi->obi;
 	isec.orig.face = shi->vlr;
-	isec.skip = RE_SKIP_VLR_NEIGHBOUR | RE_SKIP_VLR_RENDER_CHECK;
+	isec.check = RE_CHECK_VLR_RENDER;
+	isec.skip = RE_SKIP_VLR_NEIGHBOUR;
 	isec.hint = 0;
 
 	isec.hit.ob   = 0;
@@ -2209,7 +2212,8 @@ static void ray_shadow_qmc(ShadeInput *shi, LampRen *lar, float *lampco, float *
 	RE_rayobject_hint_bb( R.raytree, &bb_hint, min, max);
 	
 	isec->hint = &bb_hint;
-	isec->skip = RE_SKIP_VLR_NEIGHBOUR | RE_SKIP_VLR_RENDER_CHECK;
+	isec->check = RE_CHECK_VLR_RENDER;
+	isec->skip = RE_SKIP_VLR_NEIGHBOUR;
 	VECCOPY(vec, lampco);
 	
 	while (samples < max_samples) {
@@ -2378,7 +2382,8 @@ static void ray_shadow_jitter(ShadeInput *shi, LampRen *lar, float *lampco, floa
 		isec->vec[1] = vec[1]+lampco[1]-isec->start[1];
 		isec->vec[2] = vec[2]+lampco[2]-isec->start[2];
 		isec->labda = 1.0f;
-		isec->skip = RE_SKIP_VLR_NEIGHBOUR | RE_SKIP_VLR_RENDER_CHECK;
+		isec->check = RE_CHECK_VLR_RENDER;
+		isec->skip = RE_SKIP_VLR_NEIGHBOUR;
 		
 		if(isec->mode==RE_RAY_SHADOW_TRA) {
 			/* isec.col is like shadfac, so defines amount of light (0.0 is full shadow) */
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 37bd823fa23..4ee62bc5794 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -2422,7 +2422,8 @@ static void do_bake_shade(void *handle, int x, int y, float u, float v)
 			isec.orig.ob   = obi;
 			isec.orig.face = vlr;
 			isec.userdata= bs->actob;
-			isec.skip = RE_SKIP_VLR_NEIGHBOUR|RE_SKIP_VLR_BAKE_CHECK;
+			isec.check = RE_CHECK_VLR_BAKE;
+			isec.skip = RE_SKIP_VLR_NEIGHBOUR;
 			
 			if(bake_intersect_tree(R.raytree, &isec, shi->co, shi->vn, sign, co, &dist)) {
 				if(!hit || len_v3v3(shi->co, co) < len_v3v3(shi->co, minco)) {
diff --git a/source/blender/render/intern/source/volumetric.c b/source/blender/render/intern/source/volumetric.c
index 94cd94ff391..58ff7c5fa7e 100644
--- a/source/blender/render/intern/source/volumetric.c
+++ b/source/blender/render/intern/source/volumetric.c
@@ -93,7 +93,8 @@ static float vol_get_shadow(ShadeInput *shi, LampRen *lar, float *co)
 		}
 
 		is.mode = RE_RAY_MIRROR;
-		is.skip = RE_SKIP_VLR_RENDER_CHECK | RE_SKIP_VLR_NON_SOLID_MATERIAL;
+		is.check = RE_CHECK_VLR_NON_SOLID_MATERIAL;
+		is.skip = 0;
 		
 		if(lar->mode & (LA_LAYER|LA_LAYER_SHADOW))
 			is.lay= lar->lay;	
@@ -122,6 +123,7 @@ static int vol_get_bounds(ShadeInput *shi, float *co, float *vec, float *hitco,
 	isect->mode= RE_RAY_MIRROR;
 	isect->last_hit = NULL;
 	isect->lay= -1;
+	isect->check= RE_CHECK_VLR_NONE;
 	
 	if (intersect_type == VOL_BOUNDS_DEPTH) {
 		isect->skip = RE_SKIP_VLR_NEIGHBOUR;
@@ -186,6 +188,7 @@ static void vol_trace_behind(ShadeInput *shi, VlakRen *vlr, float *co, float *co
 	isect.labda = FLT_MAX;
 	
 	isect.mode= RE_RAY_MIRROR;
+	isect.check = RE_CHECK_VLR_NONE;
 	isect.skip = RE_SKIP_VLR_NEIGHBOUR;
 	isect.orig.ob = (void*) shi->obi;
 	isect.orig.face = (void*)vlr;