Sculpt: Add global automasking options for all brushes

This adds the automasking options to the Sculpt Tool options in a way
that they affect all brushes. This is more convenient when working with
some of these options while switching brushes as they don't need to be
enabled/disabled per brush.
An automasking option is enabled if it is enabled in the brush or in the
sculpt options.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7304

4 files changed, 66 insertions, 19 deletions

diff --git a/release/scripts/startup/bl_ui/space_view3d_toolbar.py b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
index 27159b6e609..351b72f79fb 100644
--- a/release/scripts/startup/bl_ui/space_view3d_toolbar.py
+++ b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
@@ -905,6 +905,13 @@ class VIEW3D_PT_sculpt_options(Panel, View3DPaintPanel):
         col = flow.column()
         col.prop(sculpt, "use_deform_only")
 
+        col = flow.column()
+        col.separator()
+        col.prop(sculpt, "use_automasking_topology")
+        col.prop(sculpt, "use_automasking_face_sets")
+        col.prop(sculpt, "use_automasking_boundary_edges")
+        col.prop(sculpt, "use_automasking_boundary_face_sets")
+
 
 class VIEW3D_PT_sculpt_options_gravity(Panel, View3DPaintPanel):
     bl_context = ".sculpt_mode"  # dot on purpose (access from topbar)
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 10d431d12e6..fede8ce97ca 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -1587,21 +1587,28 @@ static bool sculpt_brush_test_cyl(SculptBrushTest *test,
 
 /* Automasking */
 
-static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
+static bool sculpt_is_automasking_mode_enabled(const Sculpt *sd,
+                                               const Brush *br,
+                                               const eAutomasking_flag mode)
+{
+  return br->automasking_flags & mode || sd->automasking_flags & mode;
+}
+
+static bool sculpt_automasking_enabled(const Sculpt *sd, const SculptSession *ss, const Brush *br)
 {
   if (sculpt_stroke_is_dynamic_topology(ss, br)) {
     return false;
   }
-  if (br->automasking_flags & BRUSH_AUTOMASKING_TOPOLOGY) {
+  if (sculpt_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_TOPOLOGY)) {
     return true;
   }
-  if (br->automasking_flags & BRUSH_AUTOMASKING_FACE_SETS) {
+  if (sculpt_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_FACE_SETS)) {
     return true;
   }
-  if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
+  if (sculpt_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_BOUNDARY_EDGES)) {
     return true;
   }
-  if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+  if (sculpt_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS)) {
     return true;
   }
   return false;
@@ -1662,7 +1669,7 @@ static float *sculpt_topology_automasking_init(Sculpt *sd, Object *ob, float *au
   SculptSession *ss = ob->sculpt;
   Brush *brush = BKE_paint_brush(&sd->paint);
 
-  if (!sculpt_automasking_enabled(ss, brush)) {
+  if (!sculpt_automasking_enabled(sd, ss, brush)) {
     return NULL;
   }
 
@@ -1700,7 +1707,7 @@ static float *sculpt_face_sets_automasking_init(Sculpt *sd, Object *ob, float *a
   SculptSession *ss = ob->sculpt;
   Brush *brush = BKE_paint_brush(&sd->paint);
 
-  if (!sculpt_automasking_enabled(ss, brush)) {
+  if (!sculpt_automasking_enabled(sd, ss, brush)) {
     return NULL;
   }
 
@@ -1790,7 +1797,7 @@ static void sculpt_automasking_init(Sculpt *sd, Object *ob)
   Brush *brush = BKE_paint_brush(&sd->paint);
   const int totvert = SCULPT_vertex_count_get(ss);
 
-  if (!sculpt_automasking_enabled(ss, brush)) {
+  if (!sculpt_automasking_enabled(sd, ss, brush)) {
     return;
   }
 
@@ -1801,23 +1808,23 @@ static void sculpt_automasking_init(Sculpt *sd, Object *ob)
     ss->cache->automask[i] = 1.0f;
   }
 
-  if (brush->automasking_flags & BRUSH_AUTOMASKING_TOPOLOGY) {
+  if (sculpt_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_TOPOLOGY)) {
     SCULPT_vertex_random_access_init(ss);
     sculpt_topology_automasking_init(sd, ob, ss->cache->automask);
   }
-  if (brush->automasking_flags & BRUSH_AUTOMASKING_FACE_SETS) {
+  if (sculpt_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_FACE_SETS)) {
     SCULPT_vertex_random_access_init(ss);
     sculpt_face_sets_automasking_init(sd, ob, ss->cache->automask);
   }
 
-  if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
+  if (sculpt_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_BOUNDARY_EDGES)) {
     SCULPT_vertex_random_access_init(ss);
     sculpt_boundary_automasking_init(ob,
                                      AUTOMASK_INIT_BOUNDARY_EDGES,
                                      brush->automasking_boundary_edges_propagation_steps,
                                      ss->cache->automask);
   }
-  if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+  if (sculpt_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS)) {
     SCULPT_vertex_random_access_init(ss);
     sculpt_boundary_automasking_init(ob,
                                      AUTOMASK_INIT_BOUNDARY_FACE_SETS,
@@ -6293,7 +6300,7 @@ static void do_brush_action(Sculpt *sd, Object *ob, Brush *brush, UnifiedPaintSe
     }
 
     if (ss->cache->first_time && ss->cache->mirror_symmetry_pass == 0) {
-      if (sculpt_automasking_enabled(ss, brush)) {
+      if (sculpt_automasking_enabled(sd, ss, brush)) {
         sculpt_automasking_init(sd, ob);
       }
     }
@@ -7460,9 +7467,12 @@ static void sculpt_update_cache_variants(bContext *C, Sculpt *sd, Object *ob, Po
 /* Returns true if any of the smoothing modes are active (currently
  * one of smooth brush, autosmooth, mask smooth, or shift-key
  * smooth). */
-static bool sculpt_needs_connectivity_info(const Brush *brush, SculptSession *ss, int stroke_mode)
+static bool sculpt_needs_connectivity_info(const Sculpt *sd,
+                                           const Brush *brush,
+                                           SculptSession *ss,
+                                           int stroke_mode)
 {
-  if (ss && ss->pbvh && sculpt_automasking_enabled(ss, brush)) {
+  if (ss && ss->pbvh && sculpt_automasking_enabled(sd, ss, brush)) {
     return true;
   }
   return ((stroke_mode == BRUSH_STROKE_SMOOTH) || (ss && ss->cache && ss->cache->alt_smooth) ||
@@ -7478,8 +7488,9 @@ static void sculpt_stroke_modifiers_check(const bContext *C, Object *ob, const B
 {
   SculptSession *ss = ob->sculpt;
   View3D *v3d = CTX_wm_view3d(C);
+  Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
 
-  bool need_pmap = sculpt_needs_connectivity_info(brush, ss, 0);
+  bool need_pmap = sculpt_needs_connectivity_info(sd, brush, ss, 0);
   if (ss->shapekey_active || ss->deform_modifiers_active ||
       (!BKE_sculptsession_use_pbvh_draw(ob, v3d) && need_pmap)) {
     Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
@@ -7853,7 +7864,7 @@ static void sculpt_brush_stroke_init(bContext *C, wmOperator *op)
   view3d_operator_needs_opengl(C);
   sculpt_brush_init_tex(scene, sd, ss);
 
-  is_smooth = sculpt_needs_connectivity_info(brush, ss, mode);
+  is_smooth = sculpt_needs_connectivity_info(sd, brush, ss, mode);
   BKE_sculpt_update_object_for_edit(depsgraph, ob, is_smooth, need_mask);
 }
 
@@ -8153,7 +8164,7 @@ static void sculpt_stroke_done(const bContext *C, struct PaintStroke *UNUSED(str
       }
     }
 
-    if (sculpt_automasking_enabled(ss, brush)) {
+    if (sculpt_automasking_enabled(sd, ss, brush)) {
       sculpt_automasking_end(ob);
     }
 
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index 7837c1265ab..62640a2ce54 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -970,6 +970,8 @@ typedef struct Sculpt {
   // float pivot[3]; XXX not used?
   int flags;
 
+  int automasking_flags;
+
   /* Control tablet input */
   // char tablet_size, tablet_strength; XXX not used?
   int radial_symm[3];
@@ -987,7 +989,6 @@ typedef struct Sculpt {
   /** Constant detail resolution (Blender unit / constant_detail). */
   float constant_detail;
   float detail_percent;
-  char _pad[4];
 
   struct Object *gravity_object;
 } Sculpt;
diff --git a/source/blender/makesrna/intern/rna_sculpt_paint.c b/source/blender/makesrna/intern/rna_sculpt_paint.c
index 04a54063a03..f1a5cb79db4 100644
--- a/source/blender/makesrna/intern/rna_sculpt_paint.c
+++ b/source/blender/makesrna/intern/rna_sculpt_paint.c
@@ -815,6 +815,34 @@ static void rna_def_sculpt(BlenderRNA *brna)
   RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE);
   RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Sculpt_update");
 
+  prop = RNA_def_property(srna, "use_automasking_topology", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_TOPOLOGY);
+  RNA_def_property_ui_text(prop,
+                           "Topology Auto-masking",
+                           "Affect only vertices connected to the active vertex under the brush");
+  RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
+  prop = RNA_def_property(srna, "use_automasking_face_sets", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_FACE_SETS);
+  RNA_def_property_ui_text(prop,
+                           "Face Sets Auto-masking",
+                           "Affect only vertices that share Face Sets with the active vertex");
+  RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
+  prop = RNA_def_property(srna, "use_automasking_boundary_edges", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_BOUNDARY_EDGES);
+  RNA_def_property_ui_text(
+      prop, "Mesh Boundary Auto-masking", "Do not affect non manifold boundary edges");
+  RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
+  prop = RNA_def_property(srna, "use_automasking_boundary_face_sets", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(
+      prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS);
+  RNA_def_property_ui_text(prop,
+                           "Face Sets Boundary Auto-masking",
+                           "Do not affect vertices that belong to a Face Set boundary");
+  RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+
   prop = RNA_def_property(srna, "symmetrize_direction", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_items(prop, rna_enum_symmetrize_direction_items);
   RNA_def_property_ui_text(prop, "Direction", "Source and destination for symmetrize operator");