Fix T68723: GPencil - Split Select mode for Sculpt and Edit mode

Actually, the selection mode is not visible in Sculpt mode when mask is enabled, but still is used.
Also, the mode is shared between Edit mode and Sculpt mode and for meshes the selector is by mode.

This commit splits the select mode in different properties and show the selector in Sculpt mode to define the Select mode. Also, the Select Mask button has been removed and now the Select Mode buttons work equal to Meshes where the select buttons are the mask enable too.

Fixed some old code not valid detected during these changes.

Differential Revision: https://developer.blender.org/D5500

6 files changed, 65 insertions, 29 deletions

diff --git a/source/blender/editors/gpencil/annotate_paint.c b/source/blender/editors/gpencil/annotate_paint.c
index 07b61751b22..464f9856b4b 100644
--- a/source/blender/editors/gpencil/annotate_paint.c
+++ b/source/blender/editors/gpencil/annotate_paint.c
@@ -1252,15 +1252,6 @@ static void gp_paint_initstroke(tGPsdata *p, eGPencil_PaintModes paintmode, Deps
     /* Ensure active frame is set correctly... */
     p->gpf = p->gpl->actframe;
 
-    /* Restrict eraser to only affecting selected strokes, if the "selection mask" is on
-     * (though this is only available in editmode)
-     */
-    if (p->gpd->flag & GP_DATA_STROKE_EDITMODE) {
-      if (ts->gp_sculpt.flag & GP_SCULPT_SETT_FLAG_SELECT_MASK) {
-        p->flags |= GP_PAINTFLAG_SELECTMASK;
-      }
-    }
-
     if (has_layer_to_erase == false) {
       p->status = GP_STATUS_CAPTURE;
       // if (G.debug & G_DEBUG)
diff --git a/source/blender/editors/gpencil/gpencil_brush.c b/source/blender/editors/gpencil/gpencil_brush.c
index 8e4d2655ef0..4312bed657e 100644
--- a/source/blender/editors/gpencil/gpencil_brush.c
+++ b/source/blender/editors/gpencil/gpencil_brush.c
@@ -105,6 +105,7 @@ typedef struct tGP_BrushEditData {
   eGP_Sculpt_Types brush_type;
   eGP_Sculpt_Types brush_type_old;
   eGP_Sculpt_Flag flag;
+  eGP_Sculpt_SelectMaskFlag mask;
 
   /* Space Conversion Data */
   GP_SpaceConversion gsc;
@@ -1276,6 +1277,9 @@ static bool gpsculpt_brush_init(bContext *C, wmOperator *op)
   gso->sa = CTX_wm_area(C);
   gso->ar = CTX_wm_region(C);
 
+  /* save mask */
+  gso->mask = ts->gpencil_selectmode_sculpt;
+
   /* multiframe settings */
   gso->is_multiframe = (bool)GPENCIL_MULTIEDIT_SESSIONS_ON(gso->gpd);
   gso->use_multiframe_falloff = (ts->gp_sculpt.flag & GP_SCULPT_SETT_FLAG_FRAME_FALLOFF) != 0;
@@ -1486,7 +1490,7 @@ static bool gpsculpt_brush_do_stroke(tGP_BrushEditData *gso,
 
       /* Skip if neither one is selected
        * (and we are only allowed to edit/consider selected points) */
-      if ((gso->settings->flag & GP_SCULPT_SETT_FLAG_SELECT_MASK) && (!gso->is_weight_mode)) {
+      if ((GPENCIL_ANY_SCULPT_MASK(gso->mask)) && (!gso->is_weight_mode)) {
         if (!(pt1->flag & GP_SPOINT_SELECT) && !(pt2->flag & GP_SPOINT_SELECT)) {
           include_last = false;
           continue;
diff --git a/source/blender/editors/gpencil/gpencil_edit.c b/source/blender/editors/gpencil/gpencil_edit.c
index 878f7a1995b..d03c53bf9cb 100644
--- a/source/blender/editors/gpencil/gpencil_edit.c
+++ b/source/blender/editors/gpencil/gpencil_edit.c
@@ -220,6 +220,17 @@ void GPENCIL_OT_editmode_toggle(wmOperatorType *ot)
 }
 
 /* set select mode */
+static bool gpencil_selectmode_toggle_poll(bContext *C)
+{
+  /* edit only supported with grease pencil objects */
+  Object *ob = CTX_data_active_object(C);
+  if ((ob == NULL) || (ob->type != OB_GPENCIL) || (ob->mode != OB_MODE_EDIT_GPENCIL)) {
+    return false;
+  }
+
+  return ED_operator_view3d_active(C);
+}
+
 static int gpencil_selectmode_toggle_exec(bContext *C, wmOperator *op)
 {
   Scene *scene = CTX_data_scene(C);
@@ -227,7 +238,7 @@ static int gpencil_selectmode_toggle_exec(bContext *C, wmOperator *op)
   const int mode = RNA_int_get(op->ptr, "mode");
 
   /* Just set mode */
-  ts->gpencil_selectmode = mode;
+  ts->gpencil_selectmode_edit = mode;
 
   WM_main_add_notifier(NC_SCENE | ND_TOOLSETTINGS, NULL);
   DEG_id_tag_update(&scene->id, ID_RECALC_COPY_ON_WRITE);
@@ -246,7 +257,7 @@ void GPENCIL_OT_selectmode_toggle(wmOperatorType *ot)
 
   /* callbacks */
   ot->exec = gpencil_selectmode_toggle_exec;
-  ot->poll = gp_strokes_edit3d_poll;
+  ot->poll = gpencil_selectmode_toggle_poll;
 
   /* flags */
   ot->flag = OPTYPE_UNDO | OPTYPE_REGISTER;
diff --git a/source/blender/editors/gpencil/gpencil_intern.h b/source/blender/editors/gpencil/gpencil_intern.h
index 2a608d44a0b..715665fe6e9 100644
--- a/source/blender/editors/gpencil/gpencil_intern.h
+++ b/source/blender/editors/gpencil/gpencil_intern.h
@@ -635,6 +635,10 @@ struct GP_EditableStrokes_Iter {
   } \
   (void)0
 
+#define GPENCIL_ANY_SCULPT_MASK(flag) \
+  ((flag & (GP_SCULPT_MASK_SELECTMODE_POINT | GP_SCULPT_MASK_SELECTMODE_STROKE | \
+            GP_SCULPT_MASK_SELECTMODE_SEGMENT)))
+
 /* ****************************************************** */
 
 #endif /* __GPENCIL_INTERN_H__ */
diff --git a/source/blender/editors/gpencil/gpencil_paint.c b/source/blender/editors/gpencil/gpencil_paint.c
index a5425d64c2e..5fec9484f10 100644
--- a/source/blender/editors/gpencil/gpencil_paint.c
+++ b/source/blender/editors/gpencil/gpencil_paint.c
@@ -2128,15 +2128,6 @@ static void gp_paint_initstroke(tGPsdata *p, eGPencil_PaintModes paintmode, Deps
     /* Ensure this gets set... */
     p->gpf = p->gpl->actframe;
 
-    /* Restrict eraser to only affecting selected strokes, if the "selection mask" is on
-     * (though this is only available in editmode)
-     */
-    if (p->gpd->flag & GP_DATA_STROKE_EDITMODE) {
-      if (ts->gp_sculpt.flag & GP_SCULPT_SETT_FLAG_SELECT_MASK) {
-        p->flags |= GP_PAINTFLAG_SELECTMASK;
-      }
-    }
-
     if (has_layer_to_erase == false) {
       p->status = GP_STATUS_ERROR;
       return;
diff --git a/source/blender/editors/gpencil/gpencil_select.c b/source/blender/editors/gpencil/gpencil_select.c
index b09b91c6fc9..4c185b7fb8a 100644
--- a/source/blender/editors/gpencil/gpencil_select.c
+++ b/source/blender/editors/gpencil/gpencil_select.c
@@ -67,13 +67,30 @@
 /** \name Shared Utilities
  * \{ */
 
+/* Convert sculpt mask mode to Select mode */
+static int gpencil_select_mode_from_sculpt(eGP_Sculpt_SelectMaskFlag mode)
+{
+  if (mode & GP_SCULPT_MASK_SELECTMODE_POINT) {
+    return GP_SELECTMODE_POINT;
+  }
+  else if (mode & GP_SCULPT_MASK_SELECTMODE_STROKE) {
+    return GP_SELECTMODE_STROKE;
+  }
+  else if (GP_SCULPT_MASK_SELECTMODE_SEGMENT) {
+    return GP_SELECTMODE_SEGMENT;
+  }
+  else {
+    return GP_SELECTMODE_POINT;
+  }
+}
+
 static bool gpencil_select_poll(bContext *C)
 {
   bGPdata *gpd = ED_gpencil_data_get_active(C);
 
   if (GPENCIL_SCULPT_MODE(gpd)) {
     ToolSettings *ts = CTX_data_tool_settings(C);
-    if ((ts->gp_sculpt.flag & GP_SCULPT_SETT_FLAG_SELECT_MASK) == 0) {
+    if (!(GPENCIL_ANY_SCULPT_MASK(ts->gpencil_selectmode_sculpt))) {
       return false;
     }
   }
@@ -917,7 +934,11 @@ static int gpencil_circle_select_exec(bContext *C, wmOperator *op)
 {
   bGPdata *gpd = ED_gpencil_data_get_active(C);
   ToolSettings *ts = CTX_data_tool_settings(C);
-  const int selectmode = ts->gpencil_selectmode;
+  Object *ob = CTX_data_active_object(C);
+
+  const int selectmode = (ob && ob->mode == OB_MODE_SCULPT_GPENCIL) ?
+                             gpencil_select_mode_from_sculpt(ts->gpencil_selectmode_sculpt) :
+                             ts->gpencil_selectmode_edit;
   const float scale = ts->gp_sculpt.isect_threshold;
 
   /* if not edit/sculpt mode, the event is catched but not processed */
@@ -1023,12 +1044,18 @@ static int gpencil_generic_select_exec(bContext *C,
                                        GPencilTestFn is_inside_fn,
                                        void *user_data)
 {
+  Object *ob = CTX_data_active_object(C);
   bGPdata *gpd = ED_gpencil_data_get_active(C);
   ToolSettings *ts = CTX_data_tool_settings(C);
   ScrArea *sa = CTX_wm_area(C);
-  const bool strokemode = ((ts->gpencil_selectmode == GP_SELECTMODE_STROKE) &&
+
+  const short selectmode = (ob && ob->mode == OB_MODE_SCULPT_GPENCIL) ?
+                               gpencil_select_mode_from_sculpt(ts->gpencil_selectmode_sculpt) :
+                               ts->gpencil_selectmode_edit;
+
+  const bool strokemode = ((selectmode == GP_SELECTMODE_STROKE) &&
                            ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0));
-  const bool segmentmode = ((ts->gpencil_selectmode == GP_SELECTMODE_SEGMENT) &&
+  const bool segmentmode = ((selectmode == GP_SELECTMODE_SEGMENT) &&
                             ((gpd->flag & GP_DATA_STROKE_PAINTMODE) == 0));
   const eSelectOp sel_op = RNA_enum_get(op->ptr, "mode");
   const float scale = ts->gp_sculpt.isect_threshold;
@@ -1071,7 +1098,6 @@ static int gpencil_generic_select_exec(bContext *C,
     for (i = 0, pt = gps->points; i < gps->totpoints; i++, pt++) {
       /* convert point coords to screenspace */
       const bool is_inside = is_inside_fn(gps, pt, &gsc, gpstroke_iter.diff_mat, user_data);
-
       if (strokemode == false) {
         const bool is_select = (pt->flag & GP_SPOINT_SELECT) != 0;
         const int sel_op_result = ED_select_op_action_deselected(sel_op, is_select, is_inside);
@@ -1292,6 +1318,7 @@ static void deselect_all_selected(bContext *C)
 static int gpencil_select_exec(bContext *C, wmOperator *op)
 {
   ScrArea *sa = CTX_wm_area(C);
+  Object *ob = CTX_data_active_object(C);
   bGPdata *gpd = ED_gpencil_data_get_active(C);
   ToolSettings *ts = CTX_data_tool_settings(C);
   const float scale = ts->gp_sculpt.isect_threshold;
@@ -1322,8 +1349,12 @@ static int gpencil_select_exec(bContext *C, wmOperator *op)
   }
 
   /* if select mode is stroke, use whole stroke */
-  if (ts->gpencil_selectmode == GP_SELECTMODE_STROKE) {
-    whole = true;
+  if ((ob) && (ob->mode == OB_MODE_SCULPT_GPENCIL)) {
+    whole = (bool)(gpencil_select_mode_from_sculpt(ts->gpencil_selectmode_sculpt) ==
+                   GP_SELECTMODE_STROKE);
+  }
+  else {
+    whole = (bool)(ts->gpencil_selectmode_edit == GP_SELECTMODE_STROKE);
   }
 
   /* init space conversion stuff */
@@ -1424,7 +1455,11 @@ static int gpencil_select_exec(bContext *C, wmOperator *op)
       hit_stroke->flag |= GP_STROKE_SELECT;
 
       /* expand selection to segment */
-      if (ts->gpencil_selectmode == GP_SELECTMODE_SEGMENT) {
+      const short selectmode = (ob && ob->mode == OB_MODE_SCULPT_GPENCIL) ?
+                                   gpencil_select_mode_from_sculpt(ts->gpencil_selectmode_sculpt) :
+                                   ts->gpencil_selectmode_edit;
+
+      if (selectmode == GP_SELECTMODE_SEGMENT) {
         float r_hita[3], r_hitb[3];
         bool hit_select = (bool)(hit_point->flag & GP_SPOINT_SELECT);
         ED_gpencil_select_stroke_segment(