GPencil: Implement Holdout materials

This new feature allows to use the strokes as an eraser of any stroke below. This is very handy to open holes in filled areas.

After running some tests, we have decided to keep the additive effect of the holdout color. To get clean holdout areas, just move the color to black to remove any additive effect. To have additive effect can be used in situations like tint slightly a transparent window with blue to simulate the glass.

See T79878 for more details

Differential Revision: https://developer.blender.org/D8932

6 files changed, 78 insertions, 46 deletions

diff --git a/source/blender/draw/engines/gpencil/gpencil_cache_utils.c b/source/blender/draw/engines/gpencil/gpencil_cache_utils.c
index 363794e1be3..c4997dace7e 100644
--- a/source/blender/draw/engines/gpencil/gpencil_cache_utils.c
+++ b/source/blender/draw/engines/gpencil/gpencil_cache_utils.c
@@ -58,6 +58,17 @@ GPENCIL_tObject *gpencil_object_cache_add(GPENCIL_PrivateData *pd, Object *ob)
   tgp_ob->is_drawmode3d = (gpd->draw_mode == GP_DRAWMODE_3D) || pd->draw_depth_only;
   tgp_ob->object_scale = mat4_to_scale(ob->obmat);
 
+  /* Check if any material with holdout flag enabled. */
+  tgp_ob->do_mat_holdout = false;
+  for (int i = 0; i < ob->totcol; i++) {
+    MaterialGPencilStyle *gp_style = BKE_gpencil_material_settings(ob, i + 1);
+    if ((gp_style->flag & GP_MATERIAL_IS_STROKE_HOLDOUT) ||
+        ((gp_style->flag & GP_MATERIAL_IS_FILL_HOLDOUT))) {
+      tgp_ob->do_mat_holdout = true;
+      break;
+    }
+  }
+
   /* Find the normal most likely to represent the gpObject. */
   /* TODO: This does not work quite well if you use
    * strokes not aligned with the object axes. Maybe we could try to
diff --git a/source/blender/draw/engines/gpencil/gpencil_draw_data.c b/source/blender/draw/engines/gpencil/gpencil_draw_data.c
index 7faf426c4e0..89e47f88eec 100644
--- a/source/blender/draw/engines/gpencil/gpencil_draw_data.c
+++ b/source/blender/draw/engines/gpencil/gpencil_draw_data.c
@@ -237,6 +237,14 @@ GPENCIL_MaterialPool *gpencil_material_pool_create(GPENCIL_PrivateData *pd, Obje
       mat_data->flag |= GP_STROKE_OVERLAP;
     }
 
+    /* Material with holdout. */
+    if (gp_style->flag & GP_MATERIAL_IS_STROKE_HOLDOUT) {
+      mat_data->flag |= GP_STROKE_HOLDOUT;
+    }
+    if (gp_style->flag & GP_MATERIAL_IS_FILL_HOLDOUT) {
+      mat_data->flag |= GP_FILL_HOLDOUT;
+    }
+
     gp_style = gpencil_viewport_material_overrides(pd, ob, color_type, gp_style);
 
     /* Stroke Style */
diff --git a/source/blender/draw/engines/gpencil/gpencil_engine.h b/source/blender/draw/engines/gpencil/gpencil_engine.h
index 7a516b3dd6b..0922ab6552c 100644
--- a/source/blender/draw/engines/gpencil/gpencil_engine.h
+++ b/source/blender/draw/engines/gpencil/gpencil_engine.h
@@ -80,6 +80,8 @@ typedef struct gpMaterial {
 #define GP_STROKE_TEXTURE_STENCIL (1 << 4)
 #define GP_STROKE_TEXTURE_PREMUL (1 << 5)
 #define GP_STROKE_DOTS (1 << 6)
+#define GP_STROKE_HOLDOUT (1 << 7)
+#define GP_FILL_HOLDOUT (1 << 8)
 #define GP_FILL_TEXTURE_USE (1 << 10)
 #define GP_FILL_TEXTURE_PREMUL (1 << 11)
 #define GP_FILL_TEXTURE_CLIP (1 << 12)
@@ -194,6 +196,10 @@ typedef struct GPENCIL_tObject {
   float plane_mat[4][4];
 
   bool is_drawmode3d;
+
+  /* Use Material Holdout. */
+  bool do_mat_holdout;
+
 } GPENCIL_tObject;
 
 /* *********** LISTS *********** */
diff --git a/source/blender/draw/engines/gpencil/gpencil_shader_fx.c b/source/blender/draw/engines/gpencil/gpencil_shader_fx.c
index cf6e78f4702..bf146add19d 100644
--- a/source/blender/draw/engines/gpencil/gpencil_shader_fx.c
+++ b/source/blender/draw/engines/gpencil/gpencil_shader_fx.c
@@ -585,10 +585,6 @@ void gpencil_vfx_cache_populate(GPENCIL_Data *vedata, Object *ob, GPENCIL_tObjec
   bGPdata *gpd = (bGPdata *)ob->data;
   GPENCIL_FramebufferList *fbl = vedata->fbl;
   GPENCIL_PrivateData *pd = vedata->stl->pd;
-  /* If simplify enabled, nothing more to do. */
-  if (pd->simplify_fx) {
-    return;
-  }
 
   /* These may not be allocated yet, use adress of future pointer. */
   gpIterVfxData iter = {
@@ -601,44 +597,46 @@ void gpencil_vfx_cache_populate(GPENCIL_Data *vedata, Object *ob, GPENCIL_tObjec
       .target_reveal_tx = &pd->reveal_layer_tx,
       .source_reveal_tx = &pd->reveal_object_tx,
   };
-
-  LISTBASE_FOREACH (ShaderFxData *, fx, &ob->shader_fx) {
-    if (effect_is_active(gpd, fx, pd->is_viewport)) {
-      switch (fx->type) {
-        case eShaderFxType_Blur:
-          gpencil_vfx_blur((BlurShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Colorize:
-          gpencil_vfx_colorize((ColorizeShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Flip:
-          gpencil_vfx_flip((FlipShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Pixel:
-          gpencil_vfx_pixelize((PixelShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Rim:
-          gpencil_vfx_rim((RimShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Shadow:
-          gpencil_vfx_shadow((ShadowShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Glow:
-          gpencil_vfx_glow((GlowShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Swirl:
-          gpencil_vfx_swirl((SwirlShaderFxData *)fx, ob, &iter);
-          break;
-        case eShaderFxType_Wave:
-          gpencil_vfx_wave((WaveShaderFxData *)fx, ob, &iter);
-          break;
-        default:
-          break;
+  /* If simplify enabled, nothing more to do. */
+  if (!pd->simplify_fx) {
+    LISTBASE_FOREACH (ShaderFxData *, fx, &ob->shader_fx) {
+      if (effect_is_active(gpd, fx, pd->is_viewport)) {
+        switch (fx->type) {
+          case eShaderFxType_Blur:
+            gpencil_vfx_blur((BlurShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Colorize:
+            gpencil_vfx_colorize((ColorizeShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Flip:
+            gpencil_vfx_flip((FlipShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Pixel:
+            gpencil_vfx_pixelize((PixelShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Rim:
+            gpencil_vfx_rim((RimShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Shadow:
+            gpencil_vfx_shadow((ShadowShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Glow:
+            gpencil_vfx_glow((GlowShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Swirl:
+            gpencil_vfx_swirl((SwirlShaderFxData *)fx, ob, &iter);
+            break;
+          case eShaderFxType_Wave:
+            gpencil_vfx_wave((WaveShaderFxData *)fx, ob, &iter);
+            break;
+          default:
+            break;
+        }
       }
     }
   }
 
-  if (tgp_ob->vfx.first != NULL) {
+  if ((!pd->simplify_fx && tgp_ob->vfx.first != NULL) || tgp_ob->do_mat_holdout) {
     /* We need an extra pass to combine result to main buffer. */
     iter.target_fb = &fbl->gpencil_fb;
 
diff --git a/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl b/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
index e1323f01d9e..5e930af7bd7 100644
--- a/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
+++ b/source/blender/draw/engines/gpencil/shaders/gpencil_common_lib.glsl
@@ -34,6 +34,8 @@ struct gpMaterial {
 #define GP_STROKE_TEXTURE_STENCIL (1 << 4)
 #define GP_STROKE_TEXTURE_PREMUL (1 << 5)
 #define GP_STROKE_DOTS (1 << 6)
+#define GP_STROKE_HOLDOUT (1 << 7)
+#define GP_FILL_HOLDOUT (1 << 8)
 #define GP_FILL_TEXTURE_USE (1 << 10)
 #define GP_FILL_TEXTURE_PREMUL (1 << 11)
 #define GP_FILL_TEXTURE_CLIP (1 << 12)
@@ -44,7 +46,7 @@ struct gpMaterial {
 
 /* Multiline defines can crash blender with certain GPU drivers. */
 /* clang-format off */
-#define GP_FILL_FLAGS (GP_FILL_TEXTURE_USE | GP_FILL_TEXTURE_PREMUL | GP_FILL_TEXTURE_CLIP | GP_FILL_GRADIENT_USE | GP_FILL_GRADIENT_RADIAL)
+#define GP_FILL_FLAGS (GP_FILL_TEXTURE_USE | GP_FILL_TEXTURE_PREMUL | GP_FILL_TEXTURE_CLIP | GP_FILL_GRADIENT_USE | GP_FILL_GRADIENT_RADIAL | GP_FILL_HOLDOUT)
 /* clang-format on */
 
 #define GP_FLAG_TEST(flag, val) (((flag) & (val)) != 0)
diff --git a/source/blender/draw/engines/gpencil/shaders/gpencil_frag.glsl b/source/blender/draw/engines/gpencil/shaders/gpencil_frag.glsl
index d81c6f4fe0b..8cf457ee4dd 100644
--- a/source/blender/draw/engines/gpencil/shaders/gpencil_frag.glsl
+++ b/source/blender/draw/engines/gpencil/shaders/gpencil_frag.glsl
@@ -89,14 +89,21 @@ void main()
   fragColor *= stroke_round_cap_mask(
       strokePt1, strokePt2, strokeAspect, strokeThickness, strokeHardeness);
 
-  /* For compatibility with colored alpha buffer.
-   * Note that we are limited to mono-chromatic alpha blending here
-   * because of the blend equation and the limit of 1 color target
-   * when using custom color blending. */
-  revealColor = vec4(0.0, 0.0, 0.0, fragColor.a);
+  /* Holdout materials. */
+  if (GP_FLAG_TEST(matFlag, GP_STROKE_HOLDOUT | GP_FILL_HOLDOUT)) {
+    revealColor = fragColor.aaaa;
+  }
+  else {
+    /* NOT holdout materials.
+     * For compatibility with colored alpha buffer.
+     * Note that we are limited to mono-chromatic alpha blending here
+     * because of the blend equation and the limit of 1 color target
+     * when using custom color blending. */
+    revealColor = vec4(0.0, 0.0, 0.0, fragColor.a);
 
-  if (fragColor.a < 0.001) {
-    discard;
+    if (fragColor.a < 0.001) {
+      discard;
+    }
   }
 
   vec2 fb_size = max(vec2(textureSize(gpSceneDepthTexture, 0).xy),