Sculpt: Face Set Extract Operator

This implements a Face Set Extract operator, which is similar to mask
extract. This operator uses a picker to select and Face Set in the mesh
and extract the geometry directly to a new object.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D8599

8 files changed, 238 insertions, 52 deletions

diff --git a/release/scripts/startup/bl_ui/space_view3d.py b/release/scripts/startup/bl_ui/space_view3d.py
index c49f5b3e5d1..35b27940687 100644
--- a/release/scripts/startup/bl_ui/space_view3d.py
+++ b/release/scripts/startup/bl_ui/space_view3d.py
@@ -3112,6 +3112,10 @@ class VIEW3D_MT_face_sets(Menu):
 
         layout.separator()
 
+        op = layout.operator("mesh.face_set_extract", text='Extract Face Set')
+
+        layout.separator()
+
         op = layout.operator("sculpt.face_set_change_visibility", text='Invert Visible Face Sets')
         op.mode = 'INVERT'
 
diff --git a/source/blender/blenkernel/BKE_screen.h b/source/blender/blenkernel/BKE_screen.h
index 1090deae93f..0eeccf71cc6 100644
--- a/source/blender/blenkernel/BKE_screen.h
+++ b/source/blender/blenkernel/BKE_screen.h
@@ -399,6 +399,11 @@ struct ARegion *BKE_screen_find_region_xy(struct bScreen *screen,
                                           int x,
                                           int y) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1);
 
+struct ARegion *BKE_screen_find_main_region_at_xy(struct bScreen *screen,
+                                                  const int space_type,
+                                                  const int x,
+                                                  const int y);
+
 struct ScrArea *BKE_screen_find_area_from_space(struct bScreen *screen,
                                                 struct SpaceLink *sl) ATTR_WARN_UNUSED_RESULT
     ATTR_NONNULL(1, 2);
diff --git a/source/blender/blenkernel/intern/screen.c b/source/blender/blenkernel/intern/screen.c
index d56658a6709..998e94ff11d 100644
--- a/source/blender/blenkernel/intern/screen.c
+++ b/source/blender/blenkernel/intern/screen.c
@@ -1057,6 +1057,18 @@ void BKE_screen_view3d_shading_init(View3DShading *shading)
   memcpy(shading, shading_default, sizeof(*shading));
 }
 
+ARegion *BKE_screen_find_main_region_at_xy(bScreen *screen,
+                                           const int space_type,
+                                           const int x,
+                                           const int y)
+{
+  ScrArea *area = BKE_screen_find_area_xy(screen, space_type, x, y);
+  if (!area) {
+    return NULL;
+  }
+  return BKE_area_find_region_xy(area, RGN_TYPE_WINDOW, x, y);
+}
+
 /* magic zoom calculation, no idea what
  * it signifies, if you find out, tell me! -zr
  */
diff --git a/source/blender/editors/include/ED_sculpt.h b/source/blender/editors/include/ED_sculpt.h
index bfc46534b99..1175d08399e 100644
--- a/source/blender/editors/include/ED_sculpt.h
+++ b/source/blender/editors/include/ED_sculpt.h
@@ -57,6 +57,10 @@ void ED_sculpt_undo_geometry_end(struct Object *ob);
 int ED_sculpt_face_sets_find_next_available_id(struct Mesh *mesh);
 void ED_sculpt_face_sets_initialize_none_to_id(struct Mesh *mesh, const int new_id);
 
+int ED_sculpt_face_sets_active_update_and_get(struct bContext *C,
+                                              struct Object *ob,
+                                              const float mval[2]);
+
 /* Undo for changes happening on a base mesh for multires sculpting.
  * if there is no multires sculpt active regular undo is used. */
 void ED_sculpt_undo_push_multires_mesh_begin(struct bContext *C, const char *str);
diff --git a/source/blender/editors/mesh/editmesh_mask_extract.c b/source/blender/editors/mesh/editmesh_mask_extract.c
index 34fcee779de..17808a6e2dd 100644
--- a/source/blender/editors/mesh/editmesh_mask_extract.c
+++ b/source/blender/editors/mesh/editmesh_mask_extract.c
@@ -27,6 +27,8 @@
 
 #include "BLI_math.h"
 
+#include "BLT_translation.h"
+
 #include "BKE_context.h"
 #include "BKE_editmesh.h"
 #include "BKE_layer.h"
@@ -35,6 +37,7 @@
 #include "BKE_modifier.h"
 #include "BKE_paint.h"
 #include "BKE_report.h"
+#include "BKE_screen.h"
 #include "BKE_shrinkwrap.h"
 
 #include "DEG_depsgraph.h"
@@ -58,12 +61,17 @@
 
 #include "mesh_intern.h" /* own include */
 
-static bool paint_mask_extract_poll(bContext *C)
+typedef enum eGeometryExtractModeType {
+  GEOMETRY_EXTRACT_MASK = 0,
+  GEOMETRY_EXTRACT_FACE_SET = 1,
+} eGeometryExtractModeType;
+
+static bool geometry_extract_poll(bContext *C)
 {
   Object *ob = CTX_data_active_object(C);
   if (ob != NULL && ob->mode == OB_MODE_SCULPT) {
     if (ob->sculpt->bm) {
-      CTX_wm_operator_poll_msg_set(C, "The mask can not be extracted with dyntopo activated");
+      CTX_wm_operator_poll_msg_set(C, "The geometry can not be extracted with dyntopo activated");
       return false;
     }
     return ED_operator_object_active_editable_mesh(C);
@@ -71,14 +79,66 @@ static bool paint_mask_extract_poll(bContext *C)
   return false;
 }
 
-static int paint_mask_extract_exec(bContext *C, wmOperator *op)
+static void geometry_extract_tag_masked_faces(BMesh *bm, const float threshold)
+{
+  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+  const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
+
+  BMFace *f;
+  BMIter iter;
+  BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
+    bool keep_face = true;
+    BMVert *v;
+    BMIter face_iter;
+    BM_ITER_ELEM (v, &face_iter, f, BM_VERTS_OF_FACE) {
+      const float mask = BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
+      if (mask < threshold) {
+        keep_face = false;
+        break;
+      }
+    }
+    BM_elem_flag_set(f, BM_ELEM_TAG, !keep_face);
+  }
+}
+
+static void geometry_extract_tag_face_set(BMesh *bm, const int tag_face_set_id)
+{
+  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+  const int cd_face_sets_offset = CustomData_get_offset(&bm->pdata, CD_SCULPT_FACE_SETS);
+
+  BMFace *f;
+  BMIter iter;
+  BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
+    const int face_set_id = abs(BM_ELEM_CD_GET_INT(f, cd_face_sets_offset));
+    BM_elem_flag_set(f, BM_ELEM_TAG, face_set_id != tag_face_set_id);
+  }
+}
+
+typedef struct GeometryExtactParams {
+  /* For extracting Face Sets. */
+  int active_face_set;
+
+  /* For extracting Mask. */
+  float mask_threshold;
+
+  /* Common paramenters. */
+  bool add_boundary_loop;
+  int num_smooth_iterations;
+  bool apply_shrinkwrap;
+  bool add_solidify;
+} GeometryExtractParams;
+
+static int geometry_extract_apply(bContext *C,
+                                  wmOperator *op,
+                                  eGeometryExtractModeType mode,
+                                  GeometryExtractParams *params)
 {
   struct Main *bmain = CTX_data_main(C);
   Object *ob = CTX_data_active_object(C);
   View3D *v3d = CTX_wm_view3d(C);
   Scene *scene = CTX_data_scene(C);
-
   Depsgraph *depsgraph = CTX_data_depsgraph_on_load(C);
+
   ED_object_sculptmode_exit(C, depsgraph);
 
   BKE_sculpt_mask_layers_ensure(ob, NULL);
@@ -102,27 +162,20 @@ static int paint_mask_extract_exec(bContext *C, wmOperator *op)
   BMEditMesh *em = BKE_editmesh_create(bm, false);
   BMVert *v;
   BMEdge *ed;
-  BMFace *f;
   BMIter iter;
-  BMIter face_iter;
-
-  /* Delete all unmasked faces */
-  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
-  const int cd_vert_mask_offset = CustomData_get_offset(&bm->vdata, CD_PAINT_MASK);
 
-  float mask_threshold = RNA_float_get(op->ptr, "mask_threshold");
-  BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
-    bool keep_face = true;
-    BM_ITER_ELEM (v, &face_iter, f, BM_VERTS_OF_FACE) {
-      const float mask = BM_ELEM_CD_GET_FLOAT(v, cd_vert_mask_offset);
-      if (mask < mask_threshold) {
-        keep_face = false;
-        break;
-      }
+  switch (mode) {
+    case GEOMETRY_EXTRACT_MASK: {
+      geometry_extract_tag_masked_faces(bm, params->mask_threshold);
+      break;
+    }
+    case GEOMETRY_EXTRACT_FACE_SET: {
+      geometry_extract_tag_face_set(bm, params->active_face_set);
+      break;
     }
-    BM_elem_flag_set(f, BM_ELEM_TAG, !keep_face);
   }
 
+  /* Delete all tagged faces. */
   BM_mesh_delete_hflag_context(bm, BM_ELEM_TAG, DEL_FACES);
   BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
 
@@ -130,14 +183,13 @@ static int paint_mask_extract_exec(bContext *C, wmOperator *op)
     mul_v3_v3(v->co, ob->scale);
   }
 
-  if (RNA_boolean_get(op->ptr, "add_boundary_loop")) {
+  if (params->add_boundary_loop) {
     BM_ITER_MESH (ed, &iter, bm, BM_EDGES_OF_MESH) {
       BM_elem_flag_set(ed, BM_ELEM_TAG, BM_edge_is_boundary(ed));
     }
     edbm_extrude_edges_indiv(em, op, BM_ELEM_TAG, false);
 
-    int smooth_iterations = RNA_int_get(op->ptr, "smooth_iterations");
-    for (int repeat = 0; repeat < smooth_iterations; repeat++) {
+    for (int repeat = 0; repeat < params->num_smooth_iterations; repeat++) {
       BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
       BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
         BM_elem_flag_set(v, BM_ELEM_TAG, !BM_vert_is_boundary(v));
@@ -215,13 +267,13 @@ static int paint_mask_extract_exec(bContext *C, wmOperator *op)
   Mesh *new_ob_mesh = new_ob->data;
   CustomData_free_layers(&new_ob_mesh->pdata, CD_SCULPT_FACE_SETS, new_ob_mesh->totpoly);
 
-  if (RNA_boolean_get(op->ptr, "apply_shrinkwrap")) {
+  if (params->apply_shrinkwrap) {
     BKE_shrinkwrap_mesh_nearest_surface_deform(C, new_ob, ob);
   }
 
-  if (RNA_boolean_get(op->ptr, "add_solidify")) {
+  if (params->add_solidify) {
     ED_object_modifier_add(
-        op->reports, bmain, scene, new_ob, "mask_extract_solidify", eModifierType_Solidify);
+        op->reports, bmain, scene, new_ob, "geometry_extract_solidify", eModifierType_Solidify);
     SolidifyModifierData *sfmd = (SolidifyModifierData *)BKE_modifiers_findby_name(
         new_ob, "mask_extract_solidify");
     if (sfmd) {
@@ -240,37 +292,31 @@ static int paint_mask_extract_exec(bContext *C, wmOperator *op)
   return OPERATOR_FINISHED;
 }
 
-void MESH_OT_paint_mask_extract(wmOperatorType *ot)
+static int paint_mask_extract_exec(bContext *C, wmOperator *op)
 {
-  /* identifiers */
-  ot->name = "Mask Extract";
-  ot->description = "Create a new mesh object from the current paint mask";
-  ot->idname = "MESH_OT_paint_mask_extract";
-
-  /* api callbacks */
-  ot->poll = paint_mask_extract_poll;
-  ot->invoke = WM_operator_props_popup_confirm;
-  ot->exec = paint_mask_extract_exec;
+  GeometryExtractParams params;
+  params.mask_threshold = RNA_float_get(op->ptr, "mask_threshold");
+  params.num_smooth_iterations = RNA_int_get(op->ptr, "smooth_iterations");
+  params.add_boundary_loop = RNA_boolean_get(op->ptr, "add_boundary_loop");
+  params.apply_shrinkwrap = RNA_boolean_get(op->ptr, "apply_shrinkwrap");
+  params.add_solidify = RNA_boolean_get(op->ptr, "add_solidify");
+  return geometry_extract_apply(C, op, GEOMETRY_EXTRACT_MASK, &params);
+}
 
-  ot->flag = OPTYPE_REGISTER;
+static int paint_mask_extract_invoke(bContext *C, wmOperator *op, const wmEvent *e)
+{
+  return WM_operator_props_popup_confirm(C, op, e);
+}
 
-  RNA_def_float(
-      ot->srna,
-      "mask_threshold",
-      0.5f,
-      0.0f,
-      1.0f,
-      "Threshold",
-      "Minimum mask value to consider the vertex valid to extract a face from the original mesh",
-      0.0f,
-      1.0f);
-  RNA_def_boolean(ot->srna,
+static void geometry_extract_props(StructRNA *srna)
+{
+  RNA_def_boolean(srna,
                   "add_boundary_loop",
                   true,
                   "Add Boundary Loop",
                   "Add an extra edge loop to better preserve the shape when applying a "
                   "subdivision surface modifier");
-  RNA_def_int(ot->srna,
+  RNA_def_int(srna,
               "smooth_iterations",
               4,
               0,
@@ -279,18 +325,119 @@ void MESH_OT_paint_mask_extract(wmOperatorType *ot)
               "Smooth iterations applied to the extracted mesh",
               0,
               20);
-  RNA_def_boolean(ot->srna,
+  RNA_def_boolean(srna,
                   "apply_shrinkwrap",
                   true,
                   "Project to Sculpt",
                   "Project the extracted mesh into the original sculpt");
-  RNA_def_boolean(ot->srna,
+  RNA_def_boolean(srna,
                   "add_solidify",
                   true,
                   "Extract as Solid",
                   "Extract the mask as a solid object with a solidify modifier");
 }
 
+void MESH_OT_paint_mask_extract(wmOperatorType *ot)
+{
+  /* identifiers */
+  ot->name = "Mask Extract";
+  ot->description = "Create a new mesh object from the current paint mask";
+  ot->idname = "MESH_OT_paint_mask_extract";
+
+  /* api callbacks */
+  ot->poll = geometry_extract_poll;
+  ot->invoke = paint_mask_extract_invoke;
+  ot->exec = paint_mask_extract_exec;
+
+  ot->flag = OPTYPE_REGISTER;
+
+  RNA_def_float(
+      ot->srna,
+      "mask_threshold",
+      0.5f,
+      0.0f,
+      1.0f,
+      "Threshold",
+      "Minimum mask value to consider the vertex valid to extract a face from the original mesh",
+      0.0f,
+      1.0f);
+
+  geometry_extract_props(ot->srna);
+}
+
+static int face_set_extract_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(e))
+{
+  ED_workspace_status_text(C, TIP_("Click on the mesh to select a Face Set"));
+  WM_cursor_modal_set(CTX_wm_window(C), WM_CURSOR_EYEDROPPER);
+  WM_event_add_modal_handler(C, op);
+  return OPERATOR_RUNNING_MODAL;
+}
+
+static int face_set_extract_modal(bContext *C, wmOperator *op, const wmEvent *event)
+{
+  switch (event->type) {
+    case LEFTMOUSE:
+      if (event->val == KM_PRESS) {
+        WM_cursor_modal_restore(CTX_wm_window(C));
+        ED_workspace_status_text(C, NULL);
+
+        /* This modal operator uses and eyedropper to pick a Face Set from the mesh. This ensures
+         * that the mouse clicked in a viewport region and its coordinates can be used to raycast
+         * the PBVH and update the active Face Set ID. */
+        bScreen *screen = CTX_wm_screen(C);
+        ARegion *region = BKE_screen_find_main_region_at_xy(
+            screen, SPACE_VIEW3D, event->x, event->y);
+
+        if (!region) {
+          return OPERATOR_CANCELLED;
+        }
+
+        const float mval[2] = {event->x - region->winrct.xmin, event->y - region->winrct.ymin};
+
+        Object *ob = CTX_data_active_object(C);
+        const int face_set_id = ED_sculpt_face_sets_active_update_and_get(C, ob, mval);
+        if (face_set_id == SCULPT_FACE_SET_NONE) {
+          return OPERATOR_CANCELLED;
+        }
+
+        GeometryExtractParams params;
+        params.active_face_set = face_set_id;
+        params.num_smooth_iterations = 0;
+        params.add_boundary_loop = false;
+        params.apply_shrinkwrap = true;
+        params.add_solidify = true;
+        return geometry_extract_apply(C, op, GEOMETRY_EXTRACT_FACE_SET, &params);
+      }
+      break;
+
+    case RIGHTMOUSE: {
+      WM_cursor_modal_restore(CTX_wm_window(C));
+      ED_workspace_status_text(C, NULL);
+
+      return OPERATOR_CANCELLED;
+    }
+  }
+
+  return OPERATOR_RUNNING_MODAL;
+}
+
+void MESH_OT_face_set_extract(wmOperatorType *ot)
+{
+  /* identifiers */
+  ot->name = "Face Set Extract";
+  ot->description = "Create a new mesh object from the selected Face Set";
+  ot->idname = "MESH_OT_face_set_extract";
+
+  /* api callbacks */
+  ot->poll = geometry_extract_poll;
+  ot->invoke = face_set_extract_invoke;
+  ot->modal = face_set_extract_modal;
+
+  ot->flag = OPTYPE_REGISTER;
+
+  geometry_extract_props(ot->srna);
+}
+
 static void slice_paint_mask(BMesh *bm, bool invert, bool fill_holes, float mask_threshold)
 {
   BMVert *v;
@@ -451,7 +598,7 @@ void MESH_OT_paint_mask_slice(wmOperatorType *ot)
   ot->idname = "MESH_OT_paint_mask_slice";
 
   /* api callbacks */
-  ot->poll = paint_mask_extract_poll;
+  ot->poll = geometry_extract_poll;
   ot->exec = paint_mask_slice_exec;
 
   ot->flag = OPTYPE_REGISTER;
diff --git a/source/blender/editors/mesh/mesh_intern.h b/source/blender/editors/mesh/mesh_intern.h
index bb5da8f3a9c..a1172d17fb0 100644
--- a/source/blender/editors/mesh/mesh_intern.h
+++ b/source/blender/editors/mesh/mesh_intern.h
@@ -253,6 +253,7 @@ void MESH_OT_mod_weighted_strength(struct wmOperatorType *ot);
 
 /* *** editmesh_mask_extract.c *** */
 void MESH_OT_paint_mask_extract(struct wmOperatorType *ot);
+void MESH_OT_face_set_extract(struct wmOperatorType *ot);
 void MESH_OT_paint_mask_slice(struct wmOperatorType *ot);
 
 struct wmKeyMap *point_normals_modal_keymap(wmKeyConfig *keyconf);
diff --git a/source/blender/editors/mesh/mesh_ops.c b/source/blender/editors/mesh/mesh_ops.c
index ad1e91a57c0..2cf97b7235f 100644
--- a/source/blender/editors/mesh/mesh_ops.c
+++ b/source/blender/editors/mesh/mesh_ops.c
@@ -197,6 +197,7 @@ void ED_operatortypes_mesh(void)
   WM_operatortype_append(MESH_OT_symmetry_snap);
 
   WM_operatortype_append(MESH_OT_paint_mask_extract);
+  WM_operatortype_append(MESH_OT_face_set_extract);
   WM_operatortype_append(MESH_OT_paint_mask_slice);
 
   WM_operatortype_append(MESH_OT_point_normals);
diff --git a/source/blender/editors/sculpt_paint/sculpt_face_set.c b/source/blender/editors/sculpt_paint/sculpt_face_set.c
index b9265380a35..22d3cc215b7 100644
--- a/source/blender/editors/sculpt_paint/sculpt_face_set.c
+++ b/source/blender/editors/sculpt_paint/sculpt_face_set.c
@@ -102,6 +102,18 @@ void ED_sculpt_face_sets_initialize_none_to_id(struct Mesh *mesh, const int new_
   }
 }
 
+int ED_sculpt_face_sets_active_update_and_get(bContext *C, Object *ob, const float mval[2])
+{
+  SculptSession *ss = ob->sculpt;
+  if (!ss) {
+    return SCULPT_FACE_SET_NONE;
+  }
+
+  SculptCursorGeometryInfo gi;
+  SCULPT_cursor_geometry_info_update(C, &gi, mval, false);
+  return SCULPT_active_face_set_get(ss);
+}
+
 /* Draw Face Sets Brush. */
 
 static void do_draw_face_sets_brush_task_cb_ex(void *__restrict userdata,