Workbench: Shadows: Fix the remaining fail case.

Non-manifold geom was producing inverted result when the camera was inside
the shadow volume.

When rendering non manifold geometry with the depth fail method, we need to
emit the cap as if it was the same geometry with a virtual thickness.

Another way to view it is to imagine having a set of non-manifold geometry
all facing the light.

So for every tri we emit a front cap oriented toward the light and the
back cap pointing away from it (whatever the actual orientation of the tri).

Extrusion pass stay the same as the depth pass method (double the increment
if it's a manifold edge).

2 files changed, 19 insertions, 23 deletions

diff --git a/source/blender/draw/engines/workbench/shaders/workbench_shadow_caps_geom.glsl b/source/blender/draw/engines/workbench/shaders/workbench_shadow_caps_geom.glsl
index 32b31718cf8..948392bd8ee 100644
--- a/source/blender/draw/engines/workbench/shaders/workbench_shadow_caps_geom.glsl
+++ b/source/blender/draw/engines/workbench/shaders/workbench_shadow_caps_geom.glsl
@@ -6,9 +6,9 @@
 
 #ifdef DOUBLE_MANIFOLD
 #  ifdef USE_INVOC_EXT
-#    define invoc_ct 4
+#    define invoc_ct 2
 #  else
-#    define vert_ct 12
+#    define vert_ct 6
 #  endif
 #else
 #  ifdef USE_INVOC_EXT
@@ -39,17 +39,17 @@ vec4 get_pos(int v, bool backface)
 	return (backface) ? vData[v].backPosition : vData[v].frontPosition;
 }
 
-void emit_cap(const bool front)
+void emit_cap(const bool front, bool reversed)
 {
 	if (front) {
 		gl_Position = vData[0].frontPosition; EmitVertex();
-		gl_Position = vData[1].frontPosition; EmitVertex();
-		gl_Position = vData[2].frontPosition; EmitVertex();
+		gl_Position = vData[reversed ? 2 : 1].frontPosition; EmitVertex();
+		gl_Position = vData[reversed ? 1 : 2].frontPosition; EmitVertex();
 	}
 	else {
 		gl_Position = vData[0].backPosition; EmitVertex();
-		gl_Position = vData[2].backPosition; EmitVertex();
-		gl_Position = vData[1].backPosition; EmitVertex();
+		gl_Position = vData[reversed ? 1 : 2].backPosition; EmitVertex();
+		gl_Position = vData[reversed ? 2 : 1].backPosition; EmitVertex();
 	}
 	EndPrimitive();
 }
@@ -64,17 +64,22 @@ void main()
 
 	bool backface = facing > 0.0;
 
+#ifdef DOUBLE_MANIFOLD
+	/* In case of non manifold geom, we only increase/decrease
+	 * the stencil buffer by one but do every faces as they were facing the light. */
+	bool invert = backface;
+#else
+	const bool invert = false;
 	if (!backface) {
+#endif
 #ifdef USE_INVOC_EXT
 		bool do_front = (gl_InvocationID & 1) == 0;
-		emit_cap(do_front);
+		emit_cap(do_front, invert);
 #else
-		emit_cap(true);
-		emit_cap(false);
-#  ifdef DOUBLE_MANIFOLD
-		emit_cap(true);
-		emit_cap(false);
-#  endif
+		emit_cap(true, invert);
+		emit_cap(false, invert);
 #endif
+#ifndef DOUBLE_MANIFOLD
 	}
+#endif
 }
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index 042126802a5..f0052ba57fb 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -388,8 +388,6 @@ void workbench_deferred_cache_init(WORKBENCH_Data *vedata)
 			psl->shadow_depth_fail_caps_mani_pass = DRW_pass_create("Shadow Fail Caps Mani", depth_fail_state);
 
 #ifndef DEBUG_SHADOW_VOLUME
-			grp = DRW_shgroup_create(e_data.shadow_caps_sh, psl->shadow_depth_fail_caps_pass);
-			DRW_shgroup_stencil_mask(grp, 0xFF);
 			grp = DRW_shgroup_create(e_data.shadow_pass_sh, psl->shadow_depth_pass_pass);
 			DRW_shgroup_stencil_mask(grp, 0xFF);
 			grp = DRW_shgroup_create(e_data.shadow_pass_manifold_sh, psl->shadow_depth_pass_mani_pass);
@@ -628,13 +626,6 @@ void workbench_deferred_solid_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 						DRWShadingGroup *grp;
 						bool use_shadow_pass_technique = !studiolight_camera_in_object_shadow(wpd, ob, engine_object_data);
 
-						/* Unless we expose a parameter to the user, it's better to use the depth pass technique if the object is
-						 * non manifold. Exposing a switch to the user to force depth fail in this case can be beneficial for
-						 * planes and non-closed terrains. */
-						if (!is_manifold) {
-							use_shadow_pass_technique = true;
-						}
-
 						if (use_shadow_pass_technique) {
 							if (is_manifold) {
 								grp = DRW_shgroup_create(e_data.shadow_pass_manifold_sh, psl->shadow_depth_pass_mani_pass);