Workbench: Shadows: Fix cap being clipped by far plane.

This was the last remaining problem with shadow volumes (that I know of).

Only extrude until we hit the far plane.

6 files changed, 55 insertions, 10 deletions

diff --git a/source/blender/draw/engines/workbench/shaders/workbench_shadow_vert.glsl b/source/blender/draw/engines/workbench/shaders/workbench_shadow_vert.glsl
index 3a61bf0a286..50a721f948f 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_shadow_vert.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_shadow_vert.glsl
@@ -3,6 +3,7 @@
 uniform mat4 ModelViewProjectionMatrix;
 
 uniform vec3 lightDirection = vec3(0.57, 0.57, -0.57);
+uniform float lightDistance = 1e4;
 
 in vec3 pos;
 
@@ -16,5 +17,5 @@ void main()
 {
 	vData.pos = pos;
 	vData.frontPosition = ModelViewProjectionMatrix * vec4(pos, 1.0);
-	vData.backPosition  = ModelViewProjectionMatrix * vec4(pos + lightDirection * INFINITE, 1.0);
+	vData.backPosition  = ModelViewProjectionMatrix * vec4(pos + lightDirection * lightDistance, 1.0);
 }
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index f0052ba57fb..ba5ab7f1cc3 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -634,12 +634,15 @@ void workbench_deferred_solid_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 								grp = DRW_shgroup_create(e_data.shadow_pass_sh, psl->shadow_depth_pass_pass);
 							}
 							DRW_shgroup_uniform_vec3(grp, "lightDirection", engine_object_data->shadow_dir, 1);
+							DRW_shgroup_uniform_float_copy(grp, "lightDistance", 1e5f);
 							DRW_shgroup_call_add(grp, geom_shadow, ob->obmat);
 #ifdef DEBUG_SHADOW_VOLUME
 							DRW_debug_bbox(&engine_object_data->shadow_bbox, (float[4]){1.0f, 0.0f, 0.0f, 1.0f});
 #endif
 						}
 						else {
+							float extrude_distance = studiolight_object_shadow_distance(wpd, ob, engine_object_data);
+
 							/* TODO(fclem): only use caps if they are in the view frustum. */
 							const bool need_caps = true;
 							if (need_caps) {
@@ -650,6 +653,7 @@ void workbench_deferred_solid_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 									grp = DRW_shgroup_create(e_data.shadow_caps_sh, psl->shadow_depth_fail_caps_pass);
 								}
 								DRW_shgroup_uniform_vec3(grp, "lightDirection", engine_object_data->shadow_dir, 1);
+								DRW_shgroup_uniform_float_copy(grp, "lightDistance", extrude_distance);
 								DRW_shgroup_call_add(grp, DRW_cache_object_surface_get(ob), ob->obmat);
 							}
 
@@ -660,6 +664,7 @@ void workbench_deferred_solid_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 								grp = DRW_shgroup_create(e_data.shadow_fail_sh, psl->shadow_depth_fail_pass);
 							}
 							DRW_shgroup_uniform_vec3(grp, "lightDirection", engine_object_data->shadow_dir, 1);
+							DRW_shgroup_uniform_float_copy(grp, "lightDistance", extrude_distance);
 							DRW_shgroup_call_add(grp, geom_shadow, ob->obmat);
 #ifdef DEBUG_SHADOW_VOLUME
 							DRW_debug_bbox(&engine_object_data->shadow_bbox, (float[4]){0.0f, 1.0f, 0.0f, 1.0f});
diff --git a/source/blender/draw/engines/workbench/workbench_private.h b/source/blender/draw/engines/workbench/workbench_private.h
index ddcec29cd21..00cb6666430 100644
--- a/source/blender/draw/engines/workbench/workbench_private.h
+++ b/source/blender/draw/engines/workbench/workbench_private.h
@@ -156,6 +156,7 @@ typedef struct WORKBENCH_PrivateData {
 	float cached_shadow_direction[3];
 	float shadow_mat[4][4];
 	float shadow_inv[4][4];
+	float shadow_far_plane[4]; /* Far plane of the view frustum. */
 	float shadow_near_corners[4][3]; /* Near plane corners in shadow space. */
 	float shadow_near_min[3]; /* min and max of shadow_near_corners. allow fast test */
 	float shadow_near_max[3];
@@ -186,7 +187,7 @@ typedef struct WORKBENCH_ObjectData {
 	/* Accumulated recalc flags, which corresponds to ID->recalc flags. */
 	int recalc;
 	/* Shadow direction in local object space. */
-	float shadow_dir[3];
+	float shadow_dir[3], shadow_depth;
 	float shadow_min[3], shadow_max[3]; /* Min, max in shadow space */
 	BoundBox shadow_bbox;
 	bool shadow_bbox_dirty;
@@ -232,6 +233,7 @@ void workbench_material_set_normal_world_matrix(
 void studiolight_update_world(StudioLight *sl, WORKBENCH_UBO_World *wd);
 void studiolight_update_light(WORKBENCH_PrivateData *wpd, const float light_direction[3]);
 bool studiolight_object_cast_visible_shadow(WORKBENCH_PrivateData *wpd, Object *ob, WORKBENCH_ObjectData *oed);
+float studiolight_object_shadow_distance(WORKBENCH_PrivateData *wpd, Object *ob, WORKBENCH_ObjectData *oed);
 bool studiolight_camera_in_object_shadow(WORKBENCH_PrivateData *wpd, Object *ob, WORKBENCH_ObjectData *oed);
 
 /* workbench_data.c */
diff --git a/source/blender/draw/engines/workbench/workbench_studiolight.c b/source/blender/draw/engines/workbench/workbench_studiolight.c
index 4db89717b2a..6451d1f57c8 100644
--- a/source/blender/draw/engines/workbench/workbench_studiolight.c
+++ b/source/blender/draw/engines/workbench/workbench_studiolight.c
@@ -76,9 +76,13 @@ void studiolight_update_light(WORKBENCH_PrivateData *wpd, const float light_dire
 		invert_m4_m4(wpd->shadow_inv, wpd->shadow_mat);
 
 		copy_v3_v3(wpd->cached_shadow_direction, light_direction);
-
 	}
 
+	float planes[6][4];
+	DRW_culling_frustum_planes_get(planes);
+	/* we only need the far plane. */
+	copy_v4_v4(wpd->shadow_far_plane, planes[2]);
+
 	BoundBox frustum_corners;
 	DRW_culling_frustum_corners_get(&frustum_corners);
 
@@ -112,8 +116,9 @@ static BoundBox *studiolight_object_shadow_bbox_get(WORKBENCH_PrivateData *wpd,
 			mul_v3_m4v3(corner, tmp_mat, bbox->vec[i]);
 			minmax_v3v3_v3(oed->shadow_min, oed->shadow_max, corner);
 		}
+		oed->shadow_depth = oed->shadow_max[2] - oed->shadow_min[2];
 		/* Extend towards infinity. */
-		oed->shadow_max[2] += 1e4;
+		oed->shadow_max[2] += 1e4f;
 
 		/* Get extended AABB in world space. */
 		BKE_boundbox_init_from_minmax(&oed->shadow_bbox, oed->shadow_min, oed->shadow_max);
@@ -131,6 +136,30 @@ bool studiolight_object_cast_visible_shadow(WORKBENCH_PrivateData *wpd, Object *
 	return DRW_culling_box_test(shadow_bbox);
 }
 
+float studiolight_object_shadow_distance(WORKBENCH_PrivateData *wpd, Object *ob, WORKBENCH_ObjectData *oed)
+{
+	BoundBox *shadow_bbox = studiolight_object_shadow_bbox_get(wpd, ob, oed);
+
+	int corners[4] = {0, 3, 4, 7};
+	float dist = 1e4f, dist_isect;
+	for (int i = 0; i < 4; ++i) {
+		if (isect_ray_plane_v3(shadow_bbox->vec[corners[i]],
+		                       wpd->cached_shadow_direction,
+		                       wpd->shadow_far_plane,
+		                       &dist_isect, true))
+		{
+			if (dist_isect < dist) {
+				dist = dist_isect;
+			}
+		}
+		else {
+			/* All rays are parallels. If one fails, the other will too. */
+			break;
+		}
+	}
+	return max_ii(dist - oed->shadow_depth, 0);
+}
+
 bool studiolight_camera_in_object_shadow(WORKBENCH_PrivateData *wpd, Object *ob, WORKBENCH_ObjectData *oed)
 {
 	/* Just to be sure the min, max are updated. */
diff --git a/source/blender/draw/intern/DRW_render.h b/source/blender/draw/intern/DRW_render.h
index 081bae944d8..935a6d362f1 100644
--- a/source/blender/draw/intern/DRW_render.h
+++ b/source/blender/draw/intern/DRW_render.h
@@ -506,6 +506,7 @@ bool DRW_culling_box_test(BoundBox *bbox);
 bool DRW_culling_plane_test(float plane[4]);
 
 void DRW_culling_frustum_corners_get(BoundBox *corners);
+void DRW_culling_frustum_planes_get(float planes[6][4]);
 
 /* Selection */
 void DRW_select_load_id(uint id);
diff --git a/source/blender/draw/intern/draw_manager_exec.c b/source/blender/draw/intern/draw_manager_exec.c
index cb2cad8a36e..f6b6438395d 100644
--- a/source/blender/draw/intern/draw_manager_exec.c
+++ b/source/blender/draw/intern/draw_manager_exec.c
@@ -516,12 +516,12 @@ static void draw_clipping_setup_from_view(void)
 	for (int p = 0; p < 6; p++) {
 		int q, r;
 		switch (p) {
-			case 0:  q = 1; r = 2; break;
-			case 1:  q = 0; r = 5; break;
-			case 2:  q = 1; r = 5; break;
-			case 3:  q = 2; r = 6; break;
-			case 4:  q = 0; r = 3; break;
-			default: q = 4; r = 7; break;
+			case 0:  q = 1; r = 2; break; /* -X */
+			case 1:  q = 0; r = 5; break; /* -Y */
+			case 2:  q = 1; r = 5; break; /* +Z (far) */
+			case 3:  q = 2; r = 6; break; /* +Y */
+			case 4:  q = 0; r = 3; break; /* -Z (near) */
+			default: q = 4; r = 7; break; /* +X */
 		}
 		if (DST.frontface == GL_CW) {
 			SWAP(int, q, r);
@@ -713,6 +713,13 @@ void DRW_culling_frustum_corners_get(BoundBox *corners)
 	memcpy(corners, &DST.clipping.frustum_corners, sizeof(BoundBox));
 }
 
+/* See draw_clipping_setup_from_view() for the plane order. */
+void DRW_culling_frustum_planes_get(float planes[6][4])
+{
+	draw_clipping_setup_from_view();
+	memcpy(planes, &DST.clipping.frustum_planes, sizeof(DST.clipping.frustum_planes));
+}
+
 /** \} */
 
 /* -------------------------------------------------------------------- */