DRW: Make stencil state clearer and distinct

Write and test states are now separate and need to be explicit.

Also add asserts when trying to write without test enabled.

5 files changed, 93 insertions, 75 deletions

diff --git a/source/blender/draw/engines/eevee/eevee_materials.c b/source/blender/draw/engines/eevee/eevee_materials.c
index 5ac4545d223..f5f3a7a70e3 100644
--- a/source/blender/draw/engines/eevee/eevee_materials.c
+++ b/source/blender/draw/engines/eevee/eevee_materials.c
@@ -1078,7 +1078,7 @@ void EEVEE_materials_cache_init(EEVEE_ViewLayerData *sldata, EEVEE_Data *vedata)
 
   {
     DRWState state = (DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_EQUAL | DRW_STATE_CLIP_PLANES |
-                      DRW_STATE_WRITE_STENCIL);
+                      DRW_STATE_WRITE_STENCIL | DRW_STATE_STENCIL_ALWAYS);
     DRW_PASS_CREATE(psl->sss_pass, state);
     DRW_PASS_CREATE(psl->sss_pass_cull, state | DRW_STATE_CULL_BACK);
     e_data.sss_count = 0;
diff --git a/source/blender/draw/engines/eevee/eevee_subsurface.c b/source/blender/draw/engines/eevee/eevee_subsurface.c
index 0e451dc41da..24956239508 100644
--- a/source/blender/draw/engines/eevee/eevee_subsurface.c
+++ b/source/blender/draw/engines/eevee/eevee_subsurface.c
@@ -174,7 +174,7 @@ void EEVEE_subsurface_output_init(EEVEE_ViewLayerData *UNUSED(sldata), EEVEE_Dat
 
     /* Make the opaque refraction pass mask the sss. */
     DRWState state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_EQUAL | DRW_STATE_CLIP_PLANES |
-                     DRW_STATE_WRITE_STENCIL;
+                     DRW_STATE_WRITE_STENCIL | DRW_STATE_STENCIL_ALWAYS;
     DRW_pass_state_set(vedata->psl->refract_pass, state);
     DRW_pass_foreach_shgroup(vedata->psl->refract_pass, &set_shgrp_stencil, NULL);
   }
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index 28ac6d0c274..f2f211d1db9 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -760,8 +760,10 @@ void workbench_deferred_cache_init(WORKBENCH_Data *vedata)
       DRWState depth_fail_state = DRW_STATE_DEPTH_GREATER_EQUAL | DRW_STATE_WRITE_COLOR |
                                   DRW_STATE_BLEND_ADD;
 #else
-      DRWState depth_pass_state = DRW_STATE_DEPTH_LESS | DRW_STATE_WRITE_STENCIL_SHADOW_PASS;
-      DRWState depth_fail_state = DRW_STATE_DEPTH_LESS | DRW_STATE_WRITE_STENCIL_SHADOW_FAIL;
+      DRWState depth_pass_state = DRW_STATE_DEPTH_LESS | DRW_STATE_WRITE_STENCIL_SHADOW_PASS |
+                                  DRW_STATE_STENCIL_ALWAYS;
+      DRWState depth_fail_state = DRW_STATE_DEPTH_LESS | DRW_STATE_WRITE_STENCIL_SHADOW_FAIL |
+                                  DRW_STATE_STENCIL_ALWAYS;
 #endif
       psl->shadow_depth_pass_pass = DRW_pass_create("Shadow Pass", depth_pass_state);
       psl->shadow_depth_pass_mani_pass = DRW_pass_create("Shadow Pass Mani", depth_pass_state);
@@ -1246,11 +1248,11 @@ void workbench_deferred_draw_scene(WORKBENCH_Data *vedata)
     DRW_draw_pass(psl->shadow_depth_fail_caps_mani_pass);
 
     if (GHOST_ENABLED(psl)) {
-      /* We need to set the stencil buffer to 0 where Ghost objects
+      /* We need to set the stencil buffer to 0 where Ghost objects are
        * else they will get shadow and even badly shadowed. */
-      DRW_pass_state_set(psl->ghost_prepass_pass, DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL);
-      DRW_pass_state_set(psl->ghost_prepass_hair_pass,
-                         DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL);
+      DRWState state = DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL | DRW_STATE_STENCIL_ALWAYS;
+      DRW_pass_state_set(psl->ghost_prepass_pass, state);
+      DRW_pass_state_set(psl->ghost_prepass_hair_pass, state);
 
       DRW_draw_pass(psl->ghost_prepass_pass);
       DRW_draw_pass(psl->ghost_prepass_hair_pass);
@@ -1274,9 +1276,9 @@ void workbench_deferred_draw_scene(WORKBENCH_Data *vedata)
     GPU_framebuffer_bind(dfbl->depth_only_fb);
     GPU_framebuffer_clear_stencil(dfbl->depth_only_fb, 0xFF);
 
-    DRW_pass_state_set(psl->ghost_prepass_pass, DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL);
-    DRW_pass_state_set(psl->ghost_prepass_hair_pass,
-                       DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL);
+    DRWState state = DRW_STATE_DEPTH_EQUAL | DRW_STATE_WRITE_STENCIL | DRW_STATE_STENCIL_ALWAYS;
+    DRW_pass_state_set(psl->ghost_prepass_pass, state);
+    DRW_pass_state_set(psl->ghost_prepass_hair_pass, state);
 
     DRW_draw_pass(psl->ghost_prepass_pass);
     DRW_draw_pass(psl->ghost_prepass_hair_pass);
diff --git a/source/blender/draw/intern/DRW_render.h b/source/blender/draw/intern/DRW_render.h
index 98d4c3bfa53..d68525e62ae 100644
--- a/source/blender/draw/intern/DRW_render.h
+++ b/source/blender/draw/intern/DRW_render.h
@@ -332,26 +332,27 @@ typedef enum {
   DRW_STATE_CULL_BACK = (1 << 11),
   DRW_STATE_CULL_FRONT = (1 << 12),
   /** Stencil test */
-  DRW_STATE_STENCIL_EQUAL = (1 << 13),
-  DRW_STATE_STENCIL_NEQUAL = (1 << 14),
+  DRW_STATE_STENCIL_ALWAYS = (1 << 13),
+  DRW_STATE_STENCIL_EQUAL = (1 << 14),
+  DRW_STATE_STENCIL_NEQUAL = (1 << 15),
 
   /** Blend state */
-  DRW_STATE_BLEND_ADD = (1 << 15),
+  DRW_STATE_BLEND_ADD = (1 << 16),
   /** Same as additive but let alpha accumulate without premult. */
-  DRW_STATE_BLEND_ADD_FULL = (1 << 16),
+  DRW_STATE_BLEND_ADD_FULL = (1 << 17),
   /** Standard alpha blending. */
-  DRW_STATE_BLEND_ALPHA = (1 << 17),
+  DRW_STATE_BLEND_ALPHA = (1 << 18),
   /** Use that if color is already premult by alpha. */
-  DRW_STATE_BLEND_ALPHA_PREMUL = (1 << 18),
-  DRW_STATE_BLEND_ALPHA_UNDER_PREMUL = (1 << 19),
-  DRW_STATE_BLEND_OIT = (1 << 20),
-  DRW_STATE_BLEND_MUL = (1 << 21),
-
-  DRW_STATE_CLIP_PLANES = (1 << 22),
-  DRW_STATE_WIRE_SMOOTH = (1 << 23),
-  DRW_STATE_FIRST_VERTEX_CONVENTION = (1 << 24),
+  DRW_STATE_BLEND_ALPHA_PREMUL = (1 << 19),
+  DRW_STATE_BLEND_ALPHA_UNDER_PREMUL = (1 << 20),
+  DRW_STATE_BLEND_OIT = (1 << 21),
+  DRW_STATE_BLEND_MUL = (1 << 22),
+
+  DRW_STATE_CLIP_PLANES = (1 << 23),
+  DRW_STATE_WIRE_SMOOTH = (1 << 24),
+  DRW_STATE_FIRST_VERTEX_CONVENTION = (1 << 25),
   /** DO NOT USE. Assumed always enabled. Only used internally. */
-  DRW_STATE_PROGRAM_POINT_SIZE = (1 << 25),
+  DRW_STATE_PROGRAM_POINT_SIZE = (1 << 26),
 } DRWState;
 
 #define DRW_STATE_DEFAULT \
@@ -359,6 +360,14 @@ typedef enum {
 #define DRW_STATE_RASTERIZER_ENABLED \
   (DRW_STATE_WRITE_DEPTH | DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_STENCIL | \
    DRW_STATE_WRITE_STENCIL_SHADOW_PASS | DRW_STATE_WRITE_STENCIL_SHADOW_FAIL)
+#define DRW_STATE_DEPTH_TEST_ENABLED \
+  (DRW_STATE_DEPTH_ALWAYS | DRW_STATE_DEPTH_LESS | DRW_STATE_DEPTH_LESS_EQUAL | \
+   DRW_STATE_DEPTH_EQUAL | DRW_STATE_DEPTH_GREATER | DRW_STATE_DEPTH_GREATER_EQUAL)
+#define DRW_STATE_STENCIL_TEST_ENABLED \
+  (DRW_STATE_STENCIL_ALWAYS | DRW_STATE_STENCIL_EQUAL | DRW_STATE_STENCIL_NEQUAL)
+#define DRW_STATE_WRITE_STENCIL_ENABLED \
+  (DRW_STATE_WRITE_STENCIL | DRW_STATE_WRITE_STENCIL_SHADOW_PASS | \
+   DRW_STATE_WRITE_STENCIL_SHADOW_FAIL)
 
 typedef enum {
   DRW_ATTR_INT,
diff --git a/source/blender/draw/intern/draw_manager_exec.c b/source/blender/draw/intern/draw_manager_exec.c
index 85490e2f81d..2596570c022 100644
--- a/source/blender/draw/intern/draw_manager_exec.c
+++ b/source/blender/draw/intern/draw_manager_exec.c
@@ -79,6 +79,34 @@ void drw_state_set(DRWState state)
     }
   }
 
+  /* Stencil Write */
+  {
+    DRWState test;
+    if (CHANGED_ANY_STORE_VAR(DRW_STATE_WRITE_STENCIL_ENABLED, test)) {
+      /* Stencil Write */
+      if (test) {
+        glStencilMask(0xFF);
+        if (test & DRW_STATE_WRITE_STENCIL) {
+          glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+        }
+        else if (test & DRW_STATE_WRITE_STENCIL_SHADOW_PASS) {
+          glStencilOpSeparate(GL_BACK, GL_KEEP, GL_KEEP, GL_INCR_WRAP);
+          glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_DECR_WRAP);
+        }
+        else if (test & DRW_STATE_WRITE_STENCIL_SHADOW_FAIL) {
+          glStencilOpSeparate(GL_BACK, GL_KEEP, GL_DECR_WRAP, GL_KEEP);
+          glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_INCR_WRAP, GL_KEEP);
+        }
+        else {
+          BLI_assert(0);
+        }
+      }
+      else {
+        glStencilMask(0x00);
+      }
+    }
+  }
+
   /* Color Write */
   {
     int test;
@@ -130,10 +158,7 @@ void drw_state_set(DRWState state)
   /* Depth Test */
   {
     DRWState test;
-    if (CHANGED_ANY_STORE_VAR(DRW_STATE_DEPTH_LESS | DRW_STATE_DEPTH_LESS_EQUAL |
-                                  DRW_STATE_DEPTH_EQUAL | DRW_STATE_DEPTH_GREATER |
-                                  DRW_STATE_DEPTH_GREATER_EQUAL | DRW_STATE_DEPTH_ALWAYS,
-                              test)) {
+    if (CHANGED_ANY_STORE_VAR(DRW_STATE_DEPTH_TEST_ENABLED, test)) {
       if (test) {
         glEnable(GL_DEPTH_TEST);
 
@@ -165,6 +190,20 @@ void drw_state_set(DRWState state)
     }
   }
 
+  /* Stencil Test */
+  {
+    int test;
+    if (CHANGED_ANY_STORE_VAR(DRW_STATE_STENCIL_TEST_ENABLED, test)) {
+      DST.stencil_mask = STENCIL_UNDEFINED;
+      if (test) {
+        glEnable(GL_STENCIL_TEST);
+      }
+      else {
+        glDisable(GL_STENCIL_TEST);
+      }
+    }
+  }
+
   /* Wire Width */
   {
     int test;
@@ -264,49 +303,6 @@ void drw_state_set(DRWState state)
     }
   }
 
-  /* Stencil */
-  {
-    DRWState test;
-    if (CHANGED_ANY_STORE_VAR(DRW_STATE_WRITE_STENCIL | DRW_STATE_WRITE_STENCIL_SHADOW_PASS |
-                                  DRW_STATE_WRITE_STENCIL_SHADOW_FAIL | DRW_STATE_STENCIL_EQUAL |
-                                  DRW_STATE_STENCIL_NEQUAL,
-                              test)) {
-      if (test) {
-        glEnable(GL_STENCIL_TEST);
-        /* Stencil Write */
-        if ((state & DRW_STATE_WRITE_STENCIL) != 0) {
-          glStencilMask(0xFF);
-          glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-        }
-        else if ((state & DRW_STATE_WRITE_STENCIL_SHADOW_PASS) != 0) {
-          glStencilMask(0xFF);
-          glStencilOpSeparate(GL_BACK, GL_KEEP, GL_KEEP, GL_INCR_WRAP);
-          glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_KEEP, GL_DECR_WRAP);
-        }
-        else if ((state & DRW_STATE_WRITE_STENCIL_SHADOW_FAIL) != 0) {
-          glStencilMask(0xFF);
-          glStencilOpSeparate(GL_BACK, GL_KEEP, GL_DECR_WRAP, GL_KEEP);
-          glStencilOpSeparate(GL_FRONT, GL_KEEP, GL_INCR_WRAP, GL_KEEP);
-        }
-        /* Stencil Test */
-        else if ((state & (DRW_STATE_STENCIL_EQUAL | DRW_STATE_STENCIL_NEQUAL)) != 0) {
-          glStencilMask(0x00); /* disable write */
-          DST.stencil_mask = STENCIL_UNDEFINED;
-        }
-        else {
-          BLI_assert(0);
-        }
-      }
-      else {
-        /* disable write & test */
-        DST.stencil_mask = 0;
-        glStencilMask(0x00);
-        glStencilFunc(GL_ALWAYS, 0, 0xFF);
-        glDisable(GL_STENCIL_TEST);
-      }
-    }
-  }
-
   /* Provoking Vertex */
   {
     int test;
@@ -331,11 +327,9 @@ static void drw_stencil_set(uint mask)
 {
   if (DST.stencil_mask != mask) {
     DST.stencil_mask = mask;
-    /* Stencil Write */
-    if ((DST.state & DRW_STATE_WRITE_STENCIL) != 0) {
+    if ((DST.state & DRW_STATE_STENCIL_ALWAYS) != 0) {
       glStencilFunc(GL_ALWAYS, mask, 0xFF);
     }
-    /* Stencil Test */
     else if ((DST.state & DRW_STATE_STENCIL_EQUAL) != 0) {
       glStencilFunc(GL_EQUAL, mask, 0xFF);
     }
@@ -352,6 +346,18 @@ void DRW_state_reset_ex(DRWState state)
   drw_state_set(state);
 }
 
+static void drw_state_validate(void)
+{
+  /* Cannot write to stencil buffer without stencil test. */
+  if ((DST.state & DRW_STATE_WRITE_STENCIL_ENABLED)) {
+    BLI_assert(DST.state & DRW_STATE_STENCIL_TEST_ENABLED);
+  }
+  /* Cannot write to depth buffer without depth test. */
+  if ((DST.state & DRW_STATE_WRITE_DEPTH)) {
+    BLI_assert(DST.state & DRW_STATE_DEPTH_TEST_ENABLED);
+  }
+}
+
 /**
  * Use with care, intended so selection code can override passes depth settings,
  * which is important for selection to work properly.
@@ -980,6 +986,7 @@ static void drw_draw_pass_ex(DRWPass *pass,
   drw_state_set(DST.state | DRW_STATE_WRITE_DEPTH | DRW_STATE_WRITE_COLOR);
 
   drw_state_set(pass->state);
+  drw_state_validate();
 
   DRW_stats_query_start(pass->name);