GPencil: Make Sculpt Auto-masking Global and not by Brush

The auto-masking was working by Brush and this was very
inconvenient because it was necessary set the options by
Brush, now the options are global and can be set at once.

Also, the automa-masking now works with `and` logic
and not with `or` as before. That means that a stroke
must meet all the conditions of the masking.

Added new Layer and Material options to masking the 
strokes using the same Layer/Material of the selected stroke.
Before, only Active Layer and Active Material could be masked.

The options of masking has been moved to the top-bar using
the same design of Mesh Sculpt masking.

As result of the changes above, the following props changed:

Removed:

`brush.gpencil_settings.use_automasking_strokes`
`brush.gpencil_settings.use_automasking_layer`
`brush.gpencil_settings.use_automasking_material`

Added:

`tool_settings.gpencil_sculpt.use_automasking_stroke`
`tool_settings.gpencil_sculpt.use_automasking_layer_stroke`
`tool_settings.gpencil_sculpt.use_automasking_material_stroke`
`tool_settings.gpencil_sculpt.use_automasking_layer_active`
`tool_settings.gpencil_sculpt.use_automasking_material_active`


Reviewed by: Julien Kaspar, Matias Mendiola, Daniel Martinez Lara

1 files changed, 204 insertions, 94 deletions

diff --git a/source/blender/editors/gpencil/gpencil_sculpt_paint.c b/source/blender/editors/gpencil/gpencil_sculpt_paint.c
index 37c1524c2ff..ab3edfdd4fa 100644
--- a/source/blender/editors/gpencil/gpencil_sculpt_paint.c
+++ b/source/blender/editors/gpencil/gpencil_sculpt_paint.c
@@ -68,6 +68,8 @@
 
 #include "gpencil_intern.h"
 
+#define SEARCH_RADIUS_PIXEL 20
+
 /* ************************************************ */
 /* General Brush Editing Context */
 
@@ -78,6 +80,7 @@ typedef struct tGP_BrushEditData {
   Main *bmain;
   Scene *scene;
   Object *object;
+  Object *ob_eval;
 
   ScrArea *area;
   ARegion *region;
@@ -1181,6 +1184,8 @@ static bool gpencil_sculpt_brush_init(bContext *C, wmOperator *op)
     }
     /* Check if some modifier can transform the stroke. */
     gso->is_transformed = BKE_gpencil_has_transform_modifiers(ob);
+
+    gso->ob_eval = (Object *)DEG_get_evaluated_id(gso->depsgraph, &ob->id);
   }
   else {
     unit_m4(gso->inv_mat);
@@ -1196,9 +1201,13 @@ static bool gpencil_sculpt_brush_init(bContext *C, wmOperator *op)
   gso->brush = brush;
   BKE_curvemapping_init(gso->brush->curve);
 
-  if (brush->gpencil_settings->sculpt_mode_flag &
-      (GP_SCULPT_FLAGMODE_AUTOMASK_STROKE | GP_SCULPT_FLAGMODE_AUTOMASK_LAYER |
-       GP_SCULPT_FLAGMODE_AUTOMASK_MATERIAL)) {
+  const bool is_automasking = (ts->gp_sculpt.flag &
+                               (GP_SCULPT_SETT_FLAG_AUTOMASK_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_ACTIVE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_ACTIVE)) != 0;
+  if (is_automasking) {
     gso->automasking_strokes = BLI_ghash_ptr_new(__func__);
   }
   else {
@@ -1604,13 +1613,16 @@ static bool gpencil_sculpt_brush_do_frame(bContext *C,
   bGPdata *gpd = ob->data;
   const char tool = gso->brush->gpencil_sculpt_tool;
   GP_SpaceConversion *gsc = &gso->gsc;
+  ToolSettings *ts = gso->scene->toolsettings;
   Brush *brush = gso->brush;
   const int radius = (brush->flag & GP_BRUSH_USE_PRESSURE) ? gso->brush->size * gso->pressure :
                                                              gso->brush->size;
-  const bool is_automasking = (brush->gpencil_settings->sculpt_mode_flag &
-                               (GP_SCULPT_FLAGMODE_AUTOMASK_STROKE |
-                                GP_SCULPT_FLAGMODE_AUTOMASK_LAYER |
-                                GP_SCULPT_FLAGMODE_AUTOMASK_MATERIAL)) != 0;
+  const bool is_automasking = (ts->gp_sculpt.flag &
+                               (GP_SCULPT_SETT_FLAG_AUTOMASK_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_STROKE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_ACTIVE |
+                                GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_ACTIVE)) != 0;
   /* Calc bound box matrix. */
   float bound_mat[4][4];
   BKE_gpencil_layer_transform_matrix_get(gso->depsgraph, gso->object, gpl, bound_mat);
@@ -1743,27 +1755,111 @@ static bool gpencil_sculpt_brush_do_frame(bContext *C,
   return changed;
 }
 
+/* Find the stroke nearer to the brush. */
+static void get_nearest_stroke_to_brush(tGP_BrushEditData *gso,
+                                        int mval_i[2],
+                                        bGPDlayer **r_gpl,
+                                        bGPDstroke **r_gps)
+{
+  const int radius = SEARCH_RADIUS_PIXEL;
+
+  Object *ob_eval = gso->ob_eval;
+  bGPdata *gpd = (bGPdata *)ob_eval->data;
+  GP_SpaceConversion *gsc = &gso->gsc;
+  const bool is_multiedit = (bool)GPENCIL_MULTIEDIT_SESSIONS_ON(gpd);
+  float dist = FLT_MAX;
+
+  LISTBASE_FOREACH (bGPDlayer *, gpl, &gpd->layers) {
+    if (!BKE_gpencil_layer_is_editable(gpl) || (gpl->actframe == NULL)) {
+      continue;
+    }
+    /* Calculate bound box matrix. */
+    float bound_mat[4][4];
+    BKE_gpencil_layer_transform_matrix_get(gso->depsgraph, gso->object, gpl, bound_mat);
+
+    bGPDframe *init_gpf = (is_multiedit) ? gpl->frames.first : gpl->actframe;
+    for (bGPDframe *gpf = init_gpf; gpf; gpf = gpf->next) {
+      LISTBASE_FOREACH (bGPDstroke *, gps, &gpf->strokes) {
+        bGPDstroke *gps_active = (gps->runtime.gps_orig) ? gps->runtime.gps_orig : gps;
+        if (gps->totpoints == 0) {
+          continue;
+        }
+        /* Check if the color is editable. */
+        if (ED_gpencil_stroke_material_editable(gso->object, gpl, gps) == false) {
+          continue;
+        }
+
+        /* Check if the stroke collide with brush. */
+        if (!ED_gpencil_stroke_check_collision(gsc, gps, gso->mval, radius, bound_mat)) {
+          continue;
+        }
+
+        bGPDspoint *pt;
+        int pc2D[2] = {0};
+        bGPDspoint npt;
+
+        for (int i = 0; i < gps->totpoints; i++) {
+          pt = gps->points + i;
+          gpencil_point_to_world_space(pt, bound_mat, &npt);
+          gpencil_point_to_xy(gsc, gps, &npt, &pc2D[0], &pc2D[1]);
+          float d = len_v2v2_int(mval_i, pc2D);
+          if (d < dist) {
+            dist = d;
+            *r_gpl = gpl;
+            *r_gps = gps_active;
+          }
+        }
+      }
+      /* If not multi-edit, exit loop. */
+      if (!is_multiedit) {
+        break;
+      }
+    }
+  }
+}
+
 /* Get list of Auto-Masking strokes. */
 static bool get_automasking_strokes_list(tGP_BrushEditData *gso)
 {
-  bGPdata *gpd = gso->gpd;
+  Object *ob_eval = gso->ob_eval;
+  bGPdata *gpd = (bGPdata *)ob_eval->data;
   GP_SpaceConversion *gsc = &gso->gsc;
-  Brush *brush = gso->brush;
+  ToolSettings *ts = gso->scene->toolsettings;
   Object *ob = gso->object;
-  Material *mat_active = BKE_gpencil_material(ob, ob->actcol);
+  const eGP_Sculpt_SettingsFlag flag = ts->gp_sculpt.flag;
   const bool is_multiedit = (bool)GPENCIL_MULTIEDIT_SESSIONS_ON(gpd);
-  const bool is_masking_stroke = (brush->gpencil_settings->sculpt_mode_flag &
-                                  GP_SCULPT_FLAGMODE_AUTOMASK_STROKE) != 0;
-  const bool is_masking_layer = (brush->gpencil_settings->sculpt_mode_flag &
-                                 GP_SCULPT_FLAGMODE_AUTOMASK_LAYER) != 0;
-  const bool is_masking_material = (brush->gpencil_settings->sculpt_mode_flag &
-                                    GP_SCULPT_FLAGMODE_AUTOMASK_MATERIAL) != 0;
+  const bool is_masking_stroke = (flag & GP_SCULPT_SETT_FLAG_AUTOMASK_STROKE) != 0;
+  const bool is_masking_layer_stroke = (flag & GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_STROKE) != 0;
+  const bool is_masking_material_stroke = (flag & GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_STROKE) !=
+                                          0;
+  const bool is_masking_layer_active = (flag & GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_ACTIVE) != 0;
+  const bool is_masking_material_active = (flag & GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_ACTIVE) !=
+                                          0;
   int mval_i[2];
   round_v2i_v2fl(mval_i, gso->mval);
 
   /* Define a fix number of pixel as cursor radius. */
-  const int radius = 10;
+  const int radius = SEARCH_RADIUS_PIXEL;
   bGPDlayer *gpl_active = BKE_gpencil_layer_active_get(gpd);
+  Material *mat_active = BKE_gpencil_material(ob, ob->actcol);
+
+  /* By default use active values. */
+  bGPDlayer *gpl_active_stroke = gpl_active;
+  Material *mat_active_stroke = mat_active;
+  /* Find nearest stroke to find the layer and material. */
+  if (is_masking_layer_stroke || is_masking_material_stroke) {
+    bGPDlayer *gpl_near = NULL;
+    bGPDstroke *gps_near = NULL;
+    get_nearest_stroke_to_brush(gso, mval_i, &gpl_near, &gps_near);
+    if (gps_near != NULL) {
+      if (is_masking_layer_stroke) {
+        gpl_active_stroke = gpl_near;
+      }
+      if (is_masking_material_stroke) {
+        mat_active_stroke = BKE_object_material_get(ob, gps_near->mat_nr + 1);
+      }
+    }
+  }
 
   LISTBASE_FOREACH (bGPDlayer *, gpl, &gpd->layers) {
     /* Only editable and visible layers are considered. */
@@ -1777,87 +1873,113 @@ static bool get_automasking_strokes_list(tGP_BrushEditData *gso)
     bGPDframe *init_gpf = (is_multiedit) ? gpl->frames.first : gpl->actframe;
     for (bGPDframe *gpf = init_gpf; gpf; gpf = gpf->next) {
       LISTBASE_FOREACH (bGPDstroke *, gps, &gpf->strokes) {
+        bGPDstroke *gps_active = (gps->runtime.gps_orig) ? gps->runtime.gps_orig : gps;
+        bool pick_stroke = false;
+        bool pick_layer_stroke = false;
+        bool pick_material_stroke = false;
+        bool pick_layer_active = false;
+        bool pick_material_active = false;
+
         if (gps->totpoints == 0) {
           continue;
         }
-        /* Check if the color is editable. */
+        /* Check if the material is editable. */
         if (ED_gpencil_stroke_material_editable(gso->object, gpl, gps) == false) {
           continue;
         }
 
-        /* Layer Auto-Masking. */
-        if ((is_masking_layer) && (gpl == gpl_active)) {
-          BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-          continue;
+        /* Stroke Layer Auto-Masking. */
+        if (is_masking_layer_stroke && (gpl == gpl_active_stroke)) {
+          pick_layer_stroke = true;
         }
-        /* Material Auto-Masking. */
-        if (is_masking_material) {
+        /* Active Layer Auto-Masking. */
+        if (is_masking_layer_active && (gpl == gpl_active)) {
+          pick_layer_active = true;
+        }
+        /* Stroke Material Auto-Masking. */
+        if (is_masking_material_stroke) {
           Material *mat = BKE_object_material_get(ob, gps->mat_nr + 1);
-          if (mat == mat_active) {
-            BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-            continue;
+          if (mat == mat_active_stroke) {
+            pick_material_stroke = true;
           }
         }
-
-        /* If Stroke Auto-Masking is not enabled, nothing else to do. */
-        if (!is_masking_stroke) {
-          continue;
+        /* Active Material Auto-Masking. */
+        if (is_masking_material_active) {
+          Material *mat = BKE_object_material_get(ob, gps->mat_nr + 1);
+          if (mat == mat_active) {
+            pick_material_active = true;
+          }
         }
 
         /* Check if the stroke collide with brush. */
-        if (!ED_gpencil_stroke_check_collision(gsc, gps, gso->mval, radius, bound_mat)) {
-          continue;
-        }
-
-        bGPDspoint *pt1, *pt2;
-        int pc1[2] = {0};
-        int pc2[2] = {0};
-        bGPDspoint npt;
+        if ((is_masking_stroke) &&
+            ED_gpencil_stroke_check_collision(gsc, gps, gso->mval, radius, bound_mat)) {
 
-        if (gps->totpoints == 1) {
-          gpencil_point_to_world_space(gps->points, bound_mat, &npt);
-          gpencil_point_to_xy(gsc, gps, &npt, &pc1[0], &pc1[1]);
+          bGPDspoint *pt1, *pt2;
+          int pc1[2] = {0};
+          int pc2[2] = {0};
+          bGPDspoint npt;
 
-          /* Only check if point is inside. */
-          if (len_v2v2_int(mval_i, pc1) <= radius) {
-            BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-          }
-        }
-        else {
-          /* Loop over the points in the stroke, checking for intersections
-           * - an intersection means that we touched the stroke.
-           */
-          for (int i = 0; (i + 1) < gps->totpoints; i++) {
-            /* Get points to work with. */
-            pt1 = gps->points + i;
-            pt2 = gps->points + i + 1;
-
-            /* Check first point. */
-            gpencil_point_to_world_space(pt1, bound_mat, &npt);
+          if (gps->totpoints == 1) {
+            gpencil_point_to_world_space(gps->points, bound_mat, &npt);
             gpencil_point_to_xy(gsc, gps, &npt, &pc1[0], &pc1[1]);
-            if (len_v2v2_int(mval_i, pc1) <= radius) {
-              BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-              i = gps->totpoints;
-              continue;
-            }
 
-            /* Check second point. */
-            gpencil_point_to_world_space(pt2, bound_mat, &npt);
-            gpencil_point_to_xy(gsc, gps, &npt, &pc2[0], &pc2[1]);
-            if (len_v2v2_int(mval_i, pc2) <= radius) {
-              BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-              i = gps->totpoints;
-              continue;
+            /* Only check if point is inside. */
+            if (len_v2v2_int(mval_i, pc1) <= radius) {
+              pick_stroke = true;
             }
-
-            /* Check segment. */
-            if (gpencil_stroke_inside_circle(gso->mval, radius, pc1[0], pc1[1], pc2[0], pc2[1])) {
-              BLI_ghash_insert(gso->automasking_strokes, gps, gps);
-              i = gps->totpoints;
-              continue;
+          }
+          else {
+            /* Loop over the points in the stroke, checking for intersections
+             * - an intersection means that we touched the stroke.
+             */
+            for (int i = 0; (i + 1) < gps->totpoints && !pick_stroke; i++) {
+              /* Get points to work with. */
+              pt1 = gps->points + i;
+              pt2 = gps->points + i + 1;
+
+              /* Check first point. */
+              gpencil_point_to_world_space(pt1, bound_mat, &npt);
+              gpencil_point_to_xy(gsc, gps, &npt, &pc1[0], &pc1[1]);
+              if (len_v2v2_int(mval_i, pc1) <= radius) {
+                pick_stroke = true;
+                i = gps->totpoints;
+              }
+
+              /* Check second point. */
+              gpencil_point_to_world_space(pt2, bound_mat, &npt);
+              gpencil_point_to_xy(gsc, gps, &npt, &pc2[0], &pc2[1]);
+              if (len_v2v2_int(mval_i, pc2) <= radius) {
+                pick_stroke = true;
+                i = gps->totpoints;
+              }
+
+              /* Check segment. */
+              if (!pick_stroke && gpencil_stroke_inside_circle(
+                                      gso->mval, radius, pc1[0], pc1[1], pc2[0], pc2[1])) {
+                pick_stroke = true;
+                i = gps->totpoints;
+              }
             }
           }
         }
+        /* if the stroke meets all the masking conditions, add to the hash table. */
+        if (is_masking_stroke && !pick_stroke) {
+          continue;
+        }
+        if (is_masking_layer_stroke && !pick_layer_stroke) {
+          continue;
+        }
+        if (is_masking_material_stroke && !pick_material_stroke) {
+          continue;
+        }
+        if (is_masking_layer_active && !pick_layer_active) {
+          continue;
+        }
+        if (is_masking_material_active && !pick_material_active) {
+          continue;
+        }
+        BLI_ghash_insert(gso->automasking_strokes, gps_active, gps_active);
       }
       /* If not multi-edit, exit loop. */
       if (!is_multiedit) {
@@ -1877,7 +1999,7 @@ static bool gpencil_sculpt_brush_apply_standard(bContext *C, tGP_BrushEditData *
   Object *obact = gso->object;
   bool changed = false;
 
-  Object *ob_eval = (Object *)DEG_get_evaluated_id(depsgraph, &obact->id);
+  Object *ob_eval = gso->ob_eval;
   bGPdata *gpd = (bGPdata *)ob_eval->data;
 
   /* Calculate brush-specific data which applies equally to all points */
@@ -1971,6 +2093,7 @@ static void gpencil_sculpt_brush_apply(bContext *C, wmOperator *op, PointerRNA *
 {
   tGP_BrushEditData *gso = op->customdata;
   Brush *brush = gso->brush;
+  ToolSettings *ts = gso->scene->toolsettings;
   const int radius = (brush->flag & GP_BRUSH_USE_PRESSURE) ? gso->brush->size * gso->pressure :
                                                              gso->brush->size;
   float mousef[2];
@@ -2012,9 +2135,10 @@ static void gpencil_sculpt_brush_apply(bContext *C, wmOperator *op, PointerRNA *
 
   /* Get list of Auto-Masking strokes. */
   if ((!gso->automasking_ready) &&
-      (brush->gpencil_settings->sculpt_mode_flag &
-       (GP_SCULPT_FLAGMODE_AUTOMASK_STROKE | GP_SCULPT_FLAGMODE_AUTOMASK_LAYER |
-        GP_SCULPT_FLAGMODE_AUTOMASK_MATERIAL))) {
+      (ts->gp_sculpt.flag &
+       (GP_SCULPT_SETT_FLAG_AUTOMASK_STROKE | GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_STROKE |
+        GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_STROKE | GP_SCULPT_SETT_FLAG_AUTOMASK_LAYER_ACTIVE |
+        GP_SCULPT_SETT_FLAG_AUTOMASK_MATERIAL_ACTIVE))) {
     gso->automasking_ready = get_automasking_strokes_list(gso);
   }
 
@@ -2082,20 +2206,6 @@ static void gpencil_sculpt_brush_apply_event(bContext *C, wmOperator *op, const
     if (gso->brush == NULL) {
       gso->brush = gso->brush_prev;
     }
-    Brush *brush = gso->brush;
-    if (brush->gpencil_settings->sculpt_mode_flag &
-        (GP_SCULPT_FLAGMODE_AUTOMASK_STROKE | GP_SCULPT_FLAGMODE_AUTOMASK_LAYER |
-         GP_SCULPT_FLAGMODE_AUTOMASK_MATERIAL)) {
-      if (gso->automasking_strokes == NULL) {
-        gso->automasking_strokes = BLI_ghash_ptr_new(__func__);
-      }
-    }
-    else {
-      if (gso->automasking_strokes != NULL) {
-        BLI_ghash_free(gso->automasking_strokes, NULL, NULL);
-      }
-      gso->automasking_strokes = NULL;
-    }
   }
   else {
     if (gso->brush_prev != NULL) {