GPencil: Add Fill option to determine the layers used for boundary strokes

This is a patch suggested in https://blender.community/c/rightclickselect/qggbbc/

The valid values are:

* Visible Layers.
* Active Layer.
* Layer Above active.
* Layer Below active.
* All layers Above active.
* All layers Below active.

Differential Revision: https://developer.blender.org/D8474

Some minor UI changes done in the original patch.

3 files changed, 77 insertions, 1 deletions

diff --git a/source/blender/editors/gpencil/gpencil_fill.c b/source/blender/editors/gpencil/gpencil_fill.c
index d7525c3c6a5..7db25ef817b 100644
--- a/source/blender/editors/gpencil/gpencil_fill.c
+++ b/source/blender/editors/gpencil/gpencil_fill.c
@@ -104,6 +104,8 @@ typedef struct tGPDfill {
   struct bGPdata *gpd;
   /** current material */
   struct Material *mat;
+  /** current brush */
+  struct Brush *brush;
   /** layer */
   struct bGPDlayer *gpl;
   /** frame */
@@ -233,6 +235,8 @@ static void gpencil_draw_datablock(tGPDfill *tgpf, const float ink[4])
 
   Object *ob = tgpf->ob;
   bGPdata *gpd = tgpf->gpd;
+  Brush *brush = tgpf->brush;
+  BrushGpencilSettings *brush_settings = brush->gpencil_settings;
 
   tGPDdraw tgpw;
   tgpw.rv3d = tgpf->rv3d;
@@ -249,6 +253,12 @@ static void gpencil_draw_datablock(tGPDfill *tgpf, const float ink[4])
 
   GPU_blend(true);
 
+  bGPDlayer *gpl_active = BKE_gpencil_layer_active_get(gpd);
+  BLI_assert(gpl_active != NULL);
+
+  const int gpl_active_index = BLI_findindex(&gpd->layers, gpl_active);
+  BLI_assert(gpl_active_index >= 0);
+
   LISTBASE_FOREACH (bGPDlayer *, gpl, &gpd->layers) {
     /* calculate parent position */
     BKE_gpencil_parent_matrix_get(tgpw.depsgraph, ob, gpl, tgpw.diff_mat);
@@ -258,6 +268,44 @@ static void gpencil_draw_datablock(tGPDfill *tgpf, const float ink[4])
       continue;
     }
 
+    /* Decide if layer is included or not depending of the layer mode. */
+    const int gpl_index = BLI_findindex(&gpd->layers, gpl);
+    switch (brush_settings->fill_layer_mode) {
+      case GP_FILL_GPLMODE_ACTIVE: {
+        if (gpl_index != gpl_active_index) {
+          continue;
+        }
+        break;
+      }
+      case GP_FILL_GPLMODE_ABOVE: {
+        if (gpl_index != gpl_active_index + 1) {
+          continue;
+        }
+        break;
+      }
+      case GP_FILL_GPLMODE_BELOW: {
+        if (gpl_index != gpl_active_index - 1) {
+          continue;
+        }
+        break;
+      }
+      case GP_FILL_GPLMODE_ALL_ABOVE: {
+        if (gpl_index <= gpl_active_index) {
+          continue;
+        }
+        break;
+      }
+      case GP_FILL_GPLMODE_ALL_BELOW: {
+        if (gpl_index >= gpl_active_index) {
+          continue;
+        }
+        break;
+      }
+      case GP_FILL_GPLMODE_VISIBLE:
+      default:
+        break;
+    }
+
     /* if active layer and no keyframe, create a new one */
     if (gpl == tgpf->gpl) {
       if ((gpl->actframe == NULL) || (gpl->actframe->framenum != tgpf->active_cfra)) {
@@ -1247,6 +1295,7 @@ static tGPDfill *gpencil_session_init_fill(bContext *C, wmOperator *UNUSED(op))
 
   /* save filling parameters */
   Brush *brush = BKE_paint_brush(&ts->gp_paint->paint);
+  tgpf->brush = brush;
   tgpf->flag = brush->gpencil_settings->flag;
   tgpf->fill_leak = brush->gpencil_settings->fill_leak;
   tgpf->fill_threshold = brush->gpencil_settings->fill_threshold;
diff --git a/source/blender/makesdna/DNA_brush_types.h b/source/blender/makesdna/DNA_brush_types.h
index 000b61fb39b..6d56db4b0e0 100644
--- a/source/blender/makesdna/DNA_brush_types.h
+++ b/source/blender/makesdna/DNA_brush_types.h
@@ -67,7 +67,9 @@ typedef struct BrushGpencilSettings {
   short draw_smoothlvl;
   /** Number of times to subdivide new strokes. */
   short draw_subdivide;
-  char _pad[4];
+  /** Layers used for fill. */
+  short fill_layer_mode;
+  char _pad[2];
 
   /** Factor for transparency. */
   float fill_threshold;
@@ -252,6 +254,16 @@ typedef enum eGP_FillDrawModes {
   GP_FILL_DMODE_CONTROL = 2,
 } eGP_FillDrawModes;
 
+/* BrushGpencilSettings->fill_layer_mode */
+typedef enum eGP_FillLayerModes {
+  GP_FILL_GPLMODE_VISIBLE = 0,
+  GP_FILL_GPLMODE_ACTIVE = 1,
+  GP_FILL_GPLMODE_ALL_ABOVE = 2,
+  GP_FILL_GPLMODE_ALL_BELOW = 3,
+  GP_FILL_GPLMODE_ABOVE = 4,
+  GP_FILL_GPLMODE_BELOW = 5,
+} eGP_FillLayerModes;
+
 /* BrushGpencilSettings->gp_eraser_mode */
 typedef enum eGP_BrushEraserMode {
   GP_BRUSH_ERASER_SOFT = 0,
diff --git a/source/blender/makesrna/intern/rna_brush.c b/source/blender/makesrna/intern/rna_brush.c
index efc712544ca..aa8f20d4c8a 100644
--- a/source/blender/makesrna/intern/rna_brush.c
+++ b/source/blender/makesrna/intern/rna_brush.c
@@ -243,6 +243,15 @@ static EnumPropertyItem rna_enum_gpencil_fill_draw_modes_items[] = {
     {GP_FILL_DMODE_CONTROL, "CONTROL", 0, "Edit Lines", "Use edit lines as fill boundary limits"},
     {0, NULL, 0, NULL, NULL}};
 
+static EnumPropertyItem rna_enum_gpencil_fill_layers_modes_items[] = {
+    {GP_FILL_GPLMODE_VISIBLE, "VISIBLE", 0, "Visible", "Visible layers"},
+    {GP_FILL_GPLMODE_ACTIVE, "ACTIVE", 0, "Active", "Only active layer"},
+    {GP_FILL_GPLMODE_ABOVE, "ABOVE", 0, "Layer Above", "Layer above active"},
+    {GP_FILL_GPLMODE_BELOW, "BELOW", 0, "Layer Below", "Layer below active"},
+    {GP_FILL_GPLMODE_ALL_ABOVE, "ALL_ABOVE", 0, "All Above", "All layers above active"},
+    {GP_FILL_GPLMODE_ALL_BELOW, "ALL_BELOW", 0, "All Below", "All layers below active"},
+    {0, NULL, 0, NULL, NULL}};
+
 static EnumPropertyItem rna_enum_gpencil_brush_modes_items[] = {
     {GP_BRUSH_MODE_ACTIVE, "ACTIVE", 0, "Active", "Use current mode"},
     {GP_BRUSH_MODE_MATERIAL, "MATERIAL", 0, "Material", "Use always material mode"},
@@ -1646,6 +1655,12 @@ static void rna_def_gpencil_options(BlenderRNA *brna)
   RNA_def_property_ui_text(prop, "Mode", "Mode to draw boundary limits");
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
 
+  prop = RNA_def_property(srna, "fill_layer_mode", PROP_ENUM, PROP_NONE);
+  RNA_def_property_enum_sdna(prop, NULL, "fill_layer_mode");
+  RNA_def_property_enum_items(prop, rna_enum_gpencil_fill_layers_modes_items);
+  RNA_def_property_ui_text(prop, "Layer Mode", "Layers used as boundaries");
+  RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+
   prop = RNA_def_property(srna, "brush_draw_mode", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_sdna(prop, NULL, "brush_draw_mode");
   RNA_def_property_enum_items(prop, rna_enum_gpencil_brush_modes_items);