Sculpt: Face Set Edit Operator

This operator performs an edit operation in the active face set defined
by the cursor position and updates the visibility. For now, it has a
Grow and Shrink operations, similar to Select More/Less in edit mode or
to the mask filter Grow/Shrink modes. More operations can be added in
the future.

In multires, this updates the visibility of an entire face from the base
mesh at once, which makes it very convenient to edit the visible area
without manipulating the face set directly.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D7367

4 files changed, 171 insertions, 0 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index 1e52349a942..4d30c5c7fce 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -315,6 +315,8 @@ typedef struct SculptSession {
   /* Mesh Face Sets */
   /* Total number of polys of the base mesh. */
   int totfaces;
+  /* Face sets store its visibility in the sign of the integer, using the absolute value as the
+   * Face Set ID. Positive IDs are visible, negative IDs are hidden. */
   int *face_sets;
 
   /* BMesh for dynamic topology sculpting */
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 2c050cba6b7..75c88047914 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -7907,4 +7907,5 @@ void ED_operatortypes_sculpt(void)
   WM_operatortype_append(SCULPT_OT_face_sets_randomize_colors);
   WM_operatortype_append(SCULPT_OT_face_sets_init);
   WM_operatortype_append(SCULPT_OT_cloth_filter);
+  WM_operatortype_append(SCULPT_OT_face_sets_edit);
 }
diff --git a/source/blender/editors/sculpt_paint/sculpt_face_set.c b/source/blender/editors/sculpt_paint/sculpt_face_set.c
index cbdbab14690..7bb54366204 100644
--- a/source/blender/editors/sculpt_paint/sculpt_face_set.c
+++ b/source/blender/editors/sculpt_paint/sculpt_face_set.c
@@ -978,3 +978,170 @@ void SCULPT_OT_face_sets_randomize_colors(wmOperatorType *ot)
 
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
+
+typedef enum eSculptFaceSetEditMode {
+  SCULPT_FACE_SET_EDIT_GROW = 0,
+  SCULPT_FACE_SET_EDIT_SHRINK = 1,
+} eSculptFaceSetEditMode;
+
+static EnumPropertyItem prop_sculpt_face_sets_edit_types[] = {
+    {
+        SCULPT_FACE_SET_EDIT_GROW,
+        "GROW",
+        0,
+        "Grow Face Set",
+        "Grows the Face Sets boundary by one face based on mesh topology",
+    },
+    {
+        SCULPT_FACE_SET_EDIT_SHRINK,
+        "SHRINK",
+        0,
+        "Shrink Face Set",
+        "Shrinks the Face Sets boundary by one face based on mesh topology",
+    },
+    {0, NULL, 0, NULL, NULL},
+};
+
+static void sculpt_face_set_grow(Object *ob,
+                                 SculptSession *ss,
+                                 int *prev_face_sets,
+                                 const int active_face_set_id)
+{
+  Mesh *mesh = BKE_mesh_from_object(ob);
+  for (int p = 0; p < mesh->totpoly; p++) {
+    const MPoly *c_poly = &mesh->mpoly[p];
+    for (int l = 0; l < c_poly->totloop; l++) {
+      const MLoop *c_loop = &mesh->mloop[c_poly->loopstart + l];
+      const MeshElemMap *vert_map = &ss->pmap[c_loop->v];
+      for (int i = 0; i < vert_map->count; i++) {
+        const int neighbor_face_index = vert_map->indices[i];
+        if (neighbor_face_index != p) {
+
+          if (abs(prev_face_sets[neighbor_face_index]) == active_face_set_id) {
+            ss->face_sets[p] = active_face_set_id;
+          }
+        }
+      }
+    }
+  }
+}
+
+static void sculpt_face_set_shrink(Object *ob,
+                                   SculptSession *ss,
+                                   int *prev_face_sets,
+                                   const int active_face_set_id)
+{
+  Mesh *mesh = BKE_mesh_from_object(ob);
+  for (int p = 0; p < mesh->totpoly; p++) {
+    if (abs(prev_face_sets[p]) == active_face_set_id) {
+      const MPoly *c_poly = &mesh->mpoly[p];
+      for (int l = 0; l < c_poly->totloop; l++) {
+        const MLoop *c_loop = &mesh->mloop[c_poly->loopstart + l];
+        const MeshElemMap *vert_map = &ss->pmap[c_loop->v];
+        for (int i = 0; i < vert_map->count; i++) {
+          const int neighbor_face_index = vert_map->indices[i];
+          if (neighbor_face_index != p) {
+            if (abs(prev_face_sets[neighbor_face_index]) != active_face_set_id) {
+              ss->face_sets[p] = prev_face_sets[neighbor_face_index];
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+static void sculpt_face_set_apply_edit(Object *ob, const int active_face_set_id, const int mode)
+{
+  SculptSession *ss = ob->sculpt;
+
+  int *prev_face_sets = MEM_dupallocN(ss->face_sets);
+
+  switch (mode) {
+    case SCULPT_FACE_SET_EDIT_GROW:
+      sculpt_face_set_grow(ob, ss, prev_face_sets, active_face_set_id);
+      break;
+    case SCULPT_FACE_SET_EDIT_SHRINK:
+      sculpt_face_set_shrink(ob, ss, prev_face_sets, active_face_set_id);
+      break;
+  }
+
+  MEM_SAFE_FREE(prev_face_sets);
+}
+
+static int sculpt_face_set_edit_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event))
+{
+  Object *ob = CTX_data_active_object(C);
+  SculptSession *ss = ob->sculpt;
+  ARegion *region = CTX_wm_region(C);
+  Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
+
+  const int mode = RNA_enum_get(op->ptr, "mode");
+
+  /* Dyntopo not supported. */
+  if (BKE_pbvh_type(ss->pbvh) == PBVH_BMESH) {
+    return OPERATOR_CANCELLED;
+  }
+
+  BKE_sculpt_update_object_for_edit(depsgraph, ob, true, false);
+
+  PBVH *pbvh = ob->sculpt->pbvh;
+  PBVHNode **nodes;
+  int totnode;
+  BKE_pbvh_search_gather(pbvh, NULL, NULL, &nodes, &totnode);
+
+  if (!nodes) {
+    return OPERATOR_CANCELLED;
+  }
+
+  SCULPT_undo_push_begin("face set edit");
+  SCULPT_undo_push_node(ob, nodes[0], SCULPT_UNDO_FACE_SETS);
+
+  const int active_face_set = SCULPT_active_face_set_get(ss);
+
+  sculpt_face_set_apply_edit(ob, abs(active_face_set), mode);
+
+  SCULPT_undo_push_end();
+
+  /* Sync face sets visibility and vertex visibility as now all Face Sets are visible. */
+  SCULPT_visibility_sync_all_face_sets_to_vertices(ss);
+
+  for (int i = 0; i < totnode; i++) {
+    BKE_pbvh_node_mark_update_visibility(nodes[i]);
+  }
+
+  BKE_pbvh_update_vertex_data(ss->pbvh, PBVH_UpdateVisibility);
+
+  MEM_SAFE_FREE(nodes);
+
+  if (BKE_pbvh_type(pbvh) == PBVH_FACES) {
+    BKE_mesh_flush_hidden_from_verts(ob->data);
+  }
+
+  ED_region_tag_redraw(region);
+  DEG_id_tag_update(&ob->id, ID_RECALC_SHADING);
+
+  View3D *v3d = CTX_wm_view3d(C);
+  if (!BKE_sculptsession_use_pbvh_draw(ob, v3d)) {
+    DEG_id_tag_update(&ob->id, ID_RECALC_GEOMETRY);
+  }
+
+  return OPERATOR_FINISHED;
+}
+
+void SCULPT_OT_face_sets_edit(struct wmOperatorType *ot)
+{
+  /* Identifiers. */
+  ot->name = "Edit Face Set";
+  ot->idname = "SCULPT_OT_face_set_edit";
+  ot->description = "Edits the current active Face Set";
+
+  /* Api callbacks. */
+  ot->invoke = sculpt_face_set_edit_invoke;
+  ot->poll = SCULPT_mode_poll;
+
+  ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+
+  RNA_def_enum(
+      ot->srna, "mode", prop_sculpt_face_sets_edit_types, SCULPT_FACE_SET_EDIT_GROW, "Mode", "");
+}
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index 50808b04276..0e27658e848 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -890,6 +890,7 @@ void SCULPT_OT_face_sets_randomize_colors(struct wmOperatorType *ot);
 void SCULPT_OT_face_sets_change_visibility(struct wmOperatorType *ot);
 void SCULPT_OT_face_sets_init(struct wmOperatorType *ot);
 void SCULPT_OT_face_sets_create(struct wmOperatorType *ot);
+void SCULPT_OT_face_sets_edit(struct wmOperatorType *ot);
 
 /* Transform. */
 void SCULPT_OT_set_pivot_position(struct wmOperatorType *ot);