Scultp: Face Set boundary automasking

With this brush option it is possible to mask the boundary vertices of
all face sets. This is especially useful in the cloth brush, where face
sets can be used to simulate seams between different patches of cloth
and produce different patterns and effects.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7230

4 files changed, 47 insertions, 9 deletions

diff --git a/release/scripts/startup/bl_ui/properties_paint_common.py b/release/scripts/startup/bl_ui/properties_paint_common.py
index 6e0613874d1..0b006c9cea3 100644
--- a/release/scripts/startup/bl_ui/properties_paint_common.py
+++ b/release/scripts/startup/bl_ui/properties_paint_common.py
@@ -823,8 +823,9 @@ def brush_settings_advanced(layout, context, brush, popover=False):
         # face masks automasking
         layout.prop(brush, "use_automasking_face_sets")
         
-        # boundary edges automasking
+        # boundary edges/face sets automasking
         layout.prop(brush, "use_automasking_boundary_edges")
+        layout.prop(brush, "use_automasking_boundary_face_sets")
         layout.prop(brush, "automasking_boundary_edges_propagation_steps")
 
 
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index d734d2fec8b..10d431d12e6 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -1601,6 +1601,9 @@ static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
   if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
     return true;
   }
+  if (br->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+    return true;
+  }
   return false;
 }
 
@@ -1719,9 +1722,15 @@ static float *sculpt_face_sets_automasking_init(Sculpt *sd, Object *ob, float *a
 
 #define EDGE_DISTANCE_INF -1
 
-static float *sculpt_boundary_edges_automasking_init(Object *ob,
-                                                     int propagation_steps,
-                                                     float *automask_factor)
+typedef enum eBoundaryAutomaskMode {
+  AUTOMASK_INIT_BOUNDARY_EDGES = 1,
+  AUTOMASK_INIT_BOUNDARY_FACE_SETS = 2,
+} eBoundaryAutomaskMode;
+
+static float *sculpt_boundary_automasking_init(Object *ob,
+                                               eBoundaryAutomaskMode mode,
+                                               int propagation_steps,
+                                               float *automask_factor)
 {
   SculptSession *ss = ob->sculpt;
 
@@ -1735,8 +1744,17 @@ static float *sculpt_boundary_edges_automasking_init(Object *ob,
 
   for (int i = 0; i < totvert; i++) {
     edge_distance[i] = EDGE_DISTANCE_INF;
-    if (!sculpt_vertex_is_boundary(ss, i)) {
-      edge_distance[i] = 0;
+    switch (mode) {
+      case AUTOMASK_INIT_BOUNDARY_EDGES:
+        if (!sculpt_vertex_is_boundary(ss, i)) {
+          edge_distance[i] = 0;
+        }
+        break;
+      case AUTOMASK_INIT_BOUNDARY_FACE_SETS:
+        if (!SCULPT_vertex_has_unique_face_set(ss, i)) {
+          edge_distance[i] = 0;
+        }
+        break;
     }
   }
 
@@ -1794,8 +1812,17 @@ static void sculpt_automasking_init(Sculpt *sd, Object *ob)
 
   if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
     SCULPT_vertex_random_access_init(ss);
-    sculpt_boundary_edges_automasking_init(
-        ob, brush->automasking_boundary_edges_propagation_steps, ss->cache->automask);
+    sculpt_boundary_automasking_init(ob,
+                                     AUTOMASK_INIT_BOUNDARY_EDGES,
+                                     brush->automasking_boundary_edges_propagation_steps,
+                                     ss->cache->automask);
+  }
+  if (brush->automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+    SCULPT_vertex_random_access_init(ss);
+    sculpt_boundary_automasking_init(ob,
+                                     AUTOMASK_INIT_BOUNDARY_FACE_SETS,
+                                     brush->automasking_boundary_edges_propagation_steps,
+                                     ss->cache->automask);
   }
 }
 
@@ -9871,7 +9898,8 @@ static int sculpt_mesh_filter_invoke(bContext *C, wmOperator *op, const wmEvent
     for (int i = 0; i < totvert; i++) {
       ss->filter_cache->automask[i] = 1.0f;
     }
-    sculpt_boundary_edges_automasking_init(ob, 1, ss->filter_cache->automask);
+    sculpt_boundary_automasking_init(
+        ob, AUTOMASK_INIT_BOUNDARY_EDGES, 1, ss->filter_cache->automask);
   }
 
   WM_event_add_modal_handler(C, op);
diff --git a/source/blender/makesdna/DNA_brush_types.h b/source/blender/makesdna/DNA_brush_types.h
index 95d9216f550..87f0de6daa2 100644
--- a/source/blender/makesdna/DNA_brush_types.h
+++ b/source/blender/makesdna/DNA_brush_types.h
@@ -332,6 +332,7 @@ typedef enum eAutomasking_flag {
   BRUSH_AUTOMASKING_TOPOLOGY = (1 << 0),
   BRUSH_AUTOMASKING_FACE_SETS = (1 << 1),
   BRUSH_AUTOMASKING_BOUNDARY_EDGES = (1 << 2),
+  BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS = (1 << 3),
 } eAutomasking_flag;
 
 typedef struct Brush {
diff --git a/source/blender/makesrna/intern/rna_brush.c b/source/blender/makesrna/intern/rna_brush.c
index 86958c5970a..fb87f5486f4 100644
--- a/source/blender/makesrna/intern/rna_brush.c
+++ b/source/blender/makesrna/intern/rna_brush.c
@@ -2392,6 +2392,14 @@ static void rna_def_brush(BlenderRNA *brna)
       prop, "Mesh Boundary Auto-masking", "Do not affect non manifold boundary edges");
   RNA_def_property_update(prop, 0, "rna_Brush_update");
 
+  prop = RNA_def_property(srna, "use_automasking_boundary_face_sets", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(
+      prop, NULL, "automasking_flags", BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS);
+  RNA_def_property_ui_text(prop,
+                           "Face Sets Boundary Automasking",
+                           "Do not affect vertices that belong to a Face Set boundary");
+  RNA_def_property_update(prop, 0, "rna_Brush_update");
+
   prop = RNA_def_property(srna, "use_scene_spacing", PROP_ENUM, PROP_NONE);
   RNA_def_property_enum_bitflag_sdna(prop, NULL, "flag");
   RNA_def_property_enum_items(prop, brush_spacing_unit_items);