Sculpt: Mask Extract operator

This operator extracts the paint mask to a new mesh object. It can extract the paint mask creating a boundary loop in the geometry, making it ready for adding a subdivision surface modifier.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D5384

7 files changed, 318 insertions, 4 deletions

diff --git a/source/blender/blenkernel/BKE_shrinkwrap.h b/source/blender/blenkernel/BKE_shrinkwrap.h
index 9ec75c39fcf..e3d19a3d807 100644
--- a/source/blender/blenkernel/BKE_shrinkwrap.h
+++ b/source/blender/blenkernel/BKE_shrinkwrap.h
@@ -110,6 +110,11 @@ void shrinkwrapModifier_deform(struct ShrinkwrapModifierData *smd,
                                float (*vertexCos)[3],
                                int numVerts);
 
+/* Used in editmesh_mask_extract.c to shrinkwrap the extracted mesh to the sculpt */
+void BKE_shrinkwrap_mesh_nearest_surface_deform(struct bContext *C,
+                                                struct Object *ob_source,
+                                                struct Object *ob_target);
+
 /*
  * This function casts a ray in the given BVHTree.
  * but it takes into consideration the space_transform, that is:
diff --git a/source/blender/blenkernel/intern/shrinkwrap.c b/source/blender/blenkernel/intern/shrinkwrap.c
index 014a2c6d6ac..992ceda7b74 100644
--- a/source/blender/blenkernel/intern/shrinkwrap.c
+++ b/source/blender/blenkernel/intern/shrinkwrap.c
@@ -39,6 +39,7 @@
 #include "BLI_task.h"
 #include "BLI_math_solvers.h"
 
+#include "BKE_context.h"
 #include "BKE_shrinkwrap.h"
 #include "BKE_cdderivedmesh.h"
 #include "BKE_DerivedMesh.h"
@@ -1480,3 +1481,28 @@ void shrinkwrapModifier_deform(ShrinkwrapModifierData *smd,
     ss_mesh->release(ss_mesh);
   }
 }
+
+void BKE_shrinkwrap_mesh_nearest_surface_deform(struct bContext *C,
+                                                Object *ob_source,
+                                                Object *ob_target)
+{
+  Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
+  struct Scene *sce = CTX_data_scene(C);
+  ShrinkwrapModifierData ssmd = {0};
+  ModifierEvalContext ctx = {depsgraph, ob_source, 0};
+  int totvert;
+
+  ssmd.target = ob_target;
+  ssmd.shrinkType = MOD_SHRINKWRAP_NEAREST_SURFACE;
+  ssmd.shrinkMode = MOD_SHRINKWRAP_ON_SURFACE;
+  ssmd.keepDist = 0.0f;
+
+  Mesh *src_me = ob_source->data;
+  float(*vertexCos)[3] = BKE_mesh_vert_coords_alloc(src_me, &totvert);
+
+  shrinkwrapModifier_deform(&ssmd, &ctx, sce, ob_source, src_me, NULL, -1, vertexCos, totvert);
+
+  BKE_mesh_vert_coords_apply(src_me, vertexCos);
+
+  MEM_freeN(vertexCos);
+}
diff --git a/source/blender/editors/mesh/CMakeLists.txt b/source/blender/editors/mesh/CMakeLists.txt
index d7d020ae19d..88da40b947f 100644
--- a/source/blender/editors/mesh/CMakeLists.txt
+++ b/source/blender/editors/mesh/CMakeLists.txt
@@ -55,6 +55,7 @@ set(SRC
   editmesh_knife.c
   editmesh_knife_project.c
   editmesh_loopcut.c
+  editmesh_mask_extract.c
   editmesh_path.c
   editmesh_polybuild.c
   editmesh_preselect_edgering.c
diff --git a/source/blender/editors/mesh/editmesh_extrude.c b/source/blender/editors/mesh/editmesh_extrude.c
index 5b16cfd00f5..c1c8a208471 100644
--- a/source/blender/editors/mesh/editmesh_extrude.c
+++ b/source/blender/editors/mesh/editmesh_extrude.c
@@ -145,10 +145,10 @@ static bool edbm_extrude_discrete_faces(BMEditMesh *em, wmOperator *op, const ch
 }
 
 /* extrudes individual edges */
-static bool edbm_extrude_edges_indiv(BMEditMesh *em,
-                                     wmOperator *op,
-                                     const char hflag,
-                                     const bool use_normal_flip)
+bool edbm_extrude_edges_indiv(BMEditMesh *em,
+                              wmOperator *op,
+                              const char hflag,
+                              const bool use_normal_flip)
 {
   BMesh *bm = em->bm;
   BMOperator bmop;
diff --git a/source/blender/editors/mesh/editmesh_mask_extract.c b/source/blender/editors/mesh/editmesh_mask_extract.c
new file mode 100644
index 00000000000..6d51e1d3393
--- /dev/null
+++ b/source/blender/editors/mesh/editmesh_mask_extract.c
@@ -0,0 +1,272 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2019 by Blender Foundation.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup edmesh
+ */
+
+#include "DNA_mesh_types.h"
+#include "DNA_modifier_types.h"
+#include "DNA_object_types.h"
+
+#include "BLI_math.h"
+
+#include "BKE_context.h"
+#include "BKE_editmesh.h"
+#include "BKE_layer.h"
+#include "BKE_mesh.h"
+#include "BKE_modifier.h"
+#include "BKE_paint.h"
+#include "BKE_report.h"
+#include "BKE_shrinkwrap.h"
+
+#include "DEG_depsgraph.h"
+#include "DEG_depsgraph_build.h"
+
+#include "RNA_define.h"
+#include "RNA_access.h"
+
+#include "WM_types.h"
+#include "WM_api.h"
+
+#include "ED_mesh.h"
+#include "ED_screen.h"
+#include "ED_object.h"
+#include "ED_sculpt.h"
+#include "ED_view3d.h"
+
+#include "MEM_guardedalloc.h"
+
+#include "mesh_intern.h" /* own include */
+
+static bool paint_mask_extract_poll(bContext *C)
+{
+  Object *ob = CTX_data_active_object(C);
+  if (ob->mode == OB_MODE_SCULPT) {
+    if (ob->sculpt->bm) {
+      CTX_wm_operator_poll_msg_set(C, "The mask can not be extracted with dyntopo activated.");
+      return false;
+    }
+    else {
+      return true;
+    }
+  }
+  return ED_operator_object_active_editable_mesh(C);
+}
+
+static int paint_mask_extract_exec(bContext *C, wmOperator *op)
+{
+  struct Main *bmain = CTX_data_main(C);
+  Object *ob = CTX_data_active_object(C);
+  View3D *v3d = CTX_wm_view3d(C);
+  Scene *scene = CTX_data_scene(C);
+
+  Mesh *mesh = ob->data;
+  Mesh *new_mesh = BKE_mesh_copy(bmain, mesh);
+
+  const BMAllocTemplate allocsize = BMALLOC_TEMPLATE_FROM_ME(new_mesh);
+  BMesh *bm;
+  bm = BM_mesh_create(&allocsize,
+                      &((struct BMeshCreateParams){
+                          .use_toolflags = true,
+                      }));
+
+  BM_mesh_bm_from_me(bm,
+                     new_mesh,
+                     (&(struct BMeshFromMeshParams){
+                         .calc_face_normal = true,
+                     }));
+
+  BMEditMesh *em = BKE_editmesh_create(bm, false);
+  BMVert *v;
+  BMEdge *ed;
+  BMFace *f;
+  BMIter iter;
+  BMIter face_iter;
+
+  /* Delete all unmasked faces */
+  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+
+  float mask_threshold = RNA_float_get(op->ptr, "mask_threshold");
+  BM_ITER_MESH (f, &iter, bm, BM_FACES_OF_MESH) {
+    bool delete_face = false;
+    BM_ITER_ELEM (v, &face_iter, f, BM_VERTS_OF_FACE) {
+      float mask = BM_elem_float_data_get(&bm->vdata, v, CD_PAINT_MASK);
+      delete_face = mask < mask_threshold;
+    }
+    BM_elem_flag_set(f, BM_ELEM_TAG, delete_face);
+  }
+
+  BM_mesh_delete_hflag_context(bm, BM_ELEM_TAG, DEL_FACES);
+  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+
+  if (RNA_boolean_get(op->ptr, "add_boundary_loop")) {
+    BM_ITER_MESH (ed, &iter, bm, BM_EDGES_OF_MESH) {
+      BM_elem_flag_set(ed, BM_ELEM_TAG, BM_edge_is_boundary(ed));
+    }
+    edbm_extrude_edges_indiv(em, op, BM_ELEM_TAG, false);
+
+    int smooth_iterations = RNA_int_get(op->ptr, "smooth_iterations");
+    for (int repeat = 0; repeat < smooth_iterations; repeat++) {
+      BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+      BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+        BM_elem_flag_set(v, BM_ELEM_TAG, !BM_vert_is_boundary(v));
+      }
+      for (int i = 0; i < 3; i++) {
+        if (!EDBM_op_callf(em,
+                           op,
+                           "smooth_vert verts=%hv factor=%f mirror_clip_x=%b mirror_clip_y=%b "
+                           "mirror_clip_z=%b "
+                           "clip_dist=%f use_axis_x=%b use_axis_y=%b use_axis_z=%b",
+                           BM_ELEM_TAG,
+                           1.0,
+                           false,
+                           false,
+                           false,
+                           0.1,
+                           true,
+                           true,
+                           true)) {
+          continue;
+        }
+      }
+
+      BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_TAG, false);
+      BM_ITER_MESH (v, &iter, bm, BM_VERTS_OF_MESH) {
+        BM_elem_flag_set(v, BM_ELEM_TAG, BM_vert_is_boundary(v));
+      }
+      for (int i = 0; i < 1; i++) {
+        if (!EDBM_op_callf(em,
+                           op,
+                           "smooth_vert verts=%hv factor=%f mirror_clip_x=%b mirror_clip_y=%b "
+                           "mirror_clip_z=%b "
+                           "clip_dist=%f use_axis_x=%b use_axis_y=%b use_axis_z=%b",
+                           BM_ELEM_TAG,
+                           0.5,
+                           false,
+                           false,
+                           false,
+                           0.1,
+                           true,
+                           true,
+                           true)) {
+          continue;
+        }
+      }
+    }
+  }
+
+  BM_mesh_elem_hflag_disable_all(bm, BM_VERT | BM_EDGE | BM_FACE, BM_ELEM_SELECT, false);
+  BKE_editmesh_free_derivedmesh(em);
+
+  BKE_mesh_free(new_mesh);
+  new_mesh = BKE_mesh_from_bmesh_nomain(bm,
+                                        (&(struct BMeshToMeshParams){
+                                            .calc_object_remap = false,
+                                        }));
+
+  BM_mesh_free(bm);
+
+  if (new_mesh->totvert == 0) {
+    BKE_mesh_free(new_mesh);
+    return OPERATOR_FINISHED;
+  }
+
+  ushort local_view_bits = 0;
+  if (v3d && v3d->localvd) {
+    local_view_bits = v3d->local_view_uuid;
+  }
+  Object *new_ob = ED_object_add_type(C, OB_MESH, NULL, ob->loc, ob->rot, false, local_view_bits);
+  BKE_mesh_nomain_to_mesh(new_mesh, new_ob->data, new_ob, &CD_MASK_EVERYTHING, true);
+
+  BKE_mesh_free(new_mesh);
+
+  if (RNA_boolean_get(op->ptr, "apply_shrinkwrap")) {
+    BKE_shrinkwrap_mesh_nearest_surface_deform(C, new_ob, ob);
+  }
+
+  if (RNA_boolean_get(op->ptr, "add_solidify")) {
+    ED_object_modifier_add(
+        op->reports, bmain, scene, new_ob, "mask_extract_solidify", eModifierType_Solidify);
+    SolidifyModifierData *sfmd = (SolidifyModifierData *)modifiers_findByName(
+        new_ob, "mask_extract_solidify");
+    if (sfmd) {
+      sfmd->offset = -0.05f;
+    }
+  }
+
+  WM_event_add_notifier(C, NC_OBJECT | ND_MODIFIER, new_ob);
+  BKE_mesh_batch_cache_dirty_tag(new_ob->data, BKE_MESH_BATCH_DIRTY_ALL);
+  DEG_relations_tag_update(bmain);
+  DEG_id_tag_update(&new_ob->id, ID_RECALC_GEOMETRY);
+  WM_event_add_notifier(C, NC_GEOM | ND_DATA, new_ob->data);
+
+  return OPERATOR_FINISHED;
+}
+
+void MESH_OT_paint_mask_extract(wmOperatorType *ot)
+{
+  /* identifiers */
+  ot->name = "Mask Extract";
+  ot->description = "Create a new mesh object from the current paint mask";
+  ot->idname = "MESH_OT_paint_mask_extract";
+
+  /* api callbacks */
+  ot->poll = paint_mask_extract_poll;
+  ot->invoke = WM_operator_props_popup_confirm;
+  ot->exec = paint_mask_extract_exec;
+
+  ot->flag = OPTYPE_REGISTER;
+
+  RNA_def_float(
+      ot->srna,
+      "mask_threshold",
+      0.5f,
+      0.0f,
+      1.0f,
+      "Threshold",
+      "Minimum mask value to consider the vertex valid to extract a face from the original mesh",
+      0.0f,
+      1.0f);
+  RNA_def_boolean(ot->srna,
+                  "add_boundary_loop",
+                  true,
+                  "Add Boundary Loop",
+                  "Add an extra edge loop to better preserve the shape when applying a "
+                  "subdivision surface modifier");
+  RNA_def_int(ot->srna,
+              "smooth_iterations",
+              4,
+              0,
+              INT_MAX,
+              "Smooth Iterations",
+              "Smooth iterations applied to the extracted mesh",
+              0,
+              20);
+  RNA_def_boolean(ot->srna,
+                  "apply_shrinkwrap",
+                  true,
+                  "Project to Sculpt",
+                  "Project the extracted mesh into the original sculpt");
+  RNA_def_boolean(ot->srna,
+                  "add_solidify",
+                  true,
+                  "Extract as Solid",
+                  "Extract the mask as a solid object with a solidify modifier");
+}
diff --git a/source/blender/editors/mesh/mesh_intern.h b/source/blender/editors/mesh/mesh_intern.h
index 8332cb71f95..8d340d93c0a 100644
--- a/source/blender/editors/mesh/mesh_intern.h
+++ b/source/blender/editors/mesh/mesh_intern.h
@@ -79,6 +79,11 @@ struct BMElem *EDBM_elem_from_selectmode(struct BMEditMesh *em,
 int EDBM_elem_to_index_any(struct BMEditMesh *em, struct BMElem *ele);
 struct BMElem *EDBM_elem_from_index_any(struct BMEditMesh *em, int index);
 
+bool edbm_extrude_edges_indiv(struct BMEditMesh *em,
+                              struct wmOperator *op,
+                              const char hflag,
+                              const bool use_normal_flip);
+
 /* *** editmesh_add.c *** */
 void MESH_OT_primitive_plane_add(struct wmOperatorType *ot);
 void MESH_OT_primitive_cube_add(struct wmOperatorType *ot);
@@ -246,6 +251,9 @@ void MESH_OT_average_normals(struct wmOperatorType *ot);
 void MESH_OT_smoothen_normals(struct wmOperatorType *ot);
 void MESH_OT_mod_weighted_strength(struct wmOperatorType *ot);
 
+/* *** editmesh_mask_extract.c *** */
+void MESH_OT_paint_mask_extract(struct wmOperatorType *ot);
+
 struct wmKeyMap *point_normals_modal_keymap(wmKeyConfig *keyconf);
 
 #ifdef WITH_FREESTYLE
diff --git a/source/blender/editors/mesh/mesh_ops.c b/source/blender/editors/mesh/mesh_ops.c
index 28c55afbf2e..4105f853868 100644
--- a/source/blender/editors/mesh/mesh_ops.c
+++ b/source/blender/editors/mesh/mesh_ops.c
@@ -194,6 +194,8 @@ void ED_operatortypes_mesh(void)
   WM_operatortype_append(MESH_OT_symmetrize);
   WM_operatortype_append(MESH_OT_symmetry_snap);
 
+  WM_operatortype_append(MESH_OT_paint_mask_extract);
+
   WM_operatortype_append(MESH_OT_point_normals);
   WM_operatortype_append(MESH_OT_merge_normals);
   WM_operatortype_append(MESH_OT_split_normals);