Sculpt: Location option for lasso trim and depth info

This adds a location option to the trim lasso tool to position the shape
in the middle of the volume of the object instead of in the surface
under the cursor.

{F9349724}

In order to make this work, the SCULPT_cursor_geometry_info_update can
now also get the depth of the geometry from the raycast. The depth is
calculated as the second further intersecting triangle in the raycast
over the entire mesh. This information will also be used to improve and
create new tools.

Differential Revision: https://developer.blender.org/D9622

10 files changed, 94 insertions, 19 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_cursor.c b/source/blender/editors/sculpt_paint/paint_cursor.c
index 7eb08cbabac..a575e0c86d9 100644
--- a/source/blender/editors/sculpt_paint/paint_cursor.c
+++ b/source/blender/editors/sculpt_paint/paint_cursor.c
@@ -1367,7 +1367,7 @@ static void paint_cursor_sculpt_session_update_and_init(PaintCursorContext *pcon
   pcontext->prev_active_vertex_index = ss->active_vertex_index;
   if (!ups->stroke_active) {
     pcontext->is_cursor_over_mesh = SCULPT_cursor_geometry_info_update(
-        C, &gi, mouse, (pcontext->brush->falloff_shape == PAINT_FALLOFF_SHAPE_SPHERE));
+        C, &gi, mouse, (pcontext->brush->falloff_shape == PAINT_FALLOFF_SHAPE_SPHERE), false);
     copy_v3_v3(pcontext->location, gi.location);
     copy_v3_v3(pcontext->normal, gi.normal);
   }
diff --git a/source/blender/editors/sculpt_paint/paint_mask.c b/source/blender/editors/sculpt_paint/paint_mask.c
index 17d13041f28..b3e5687a6e5 100644
--- a/source/blender/editors/sculpt_paint/paint_mask.c
+++ b/source/blender/editors/sculpt_paint/paint_mask.c
@@ -949,6 +949,24 @@ static EnumPropertyItem prop_trim_orientation_types[] = {
     {0, NULL, 0, NULL, NULL},
 };
 
+typedef enum eSculptTrimLocationType {
+  SCULPT_GESTURE_TRIM_LOCATION_DEPTH_SURFACE,
+  SCULPT_GESTURE_TRIM_LOCATION_DEPTH_VOLUME,
+} eSculptTrimLocationType;
+static EnumPropertyItem prop_trim_location_types[] = {
+    {SCULPT_GESTURE_TRIM_LOCATION_DEPTH_SURFACE,
+     "DEPTH_SURFACE",
+     0,
+     "Surface",
+     "Use the surface under the cursor to locate the trimming shape"},
+    {SCULPT_GESTURE_TRIM_LOCATION_DEPTH_VOLUME,
+     "DEPTH_VOLUME",
+     0,
+     "Volume",
+     "Use the volume of the mesh to locate the trimming shape in the center of the volume"},
+    {0, NULL, 0, NULL, NULL},
+};
+
 typedef struct SculptGestureTrimOperation {
   SculptGestureOperation op;
 
@@ -962,6 +980,7 @@ typedef struct SculptGestureTrimOperation {
 
   eSculptTrimOperationType mode;
   eSculptTrimOrientationType orientation;
+  eSculptTrimLocationType location;
 } SculptGestureTrimOperation;
 
 static void sculpt_gesture_trim_normals_update(SculptGestureContext *sgcontext)
@@ -1053,8 +1072,19 @@ static void sculpt_gesture_trim_calculate_depth(bContext *C, SculptGestureContex
 
   if (trim_operation->use_cursor_depth) {
     float world_space_gesture_initial_location[3];
-    mul_v3_m4v3(
-        world_space_gesture_initial_location, vc->obact->obmat, ss->gesture_initial_location);
+
+    switch (trim_operation->location) {
+      case SCULPT_GESTURE_TRIM_LOCATION_DEPTH_SURFACE: {
+        mul_v3_m4v3(
+            world_space_gesture_initial_location, vc->obact->obmat, ss->gesture_initial_location);
+
+      } break;
+      case SCULPT_GESTURE_TRIM_LOCATION_DEPTH_VOLUME: {
+        float center_co[3];
+        mid_v3_v3v3(center_co, ss->gesture_initial_location, ss->gesture_initial_back_location);
+        mul_v3_m4v3(world_space_gesture_initial_location, vc->obact->obmat, center_co);
+      } break;
+    }
 
     float mid_point_depth;
     if (trim_operation->orientation == SCULPT_GESTURE_TRIM_ORIENTATION_VIEW) {
@@ -1362,6 +1392,7 @@ static void sculpt_gesture_init_trim_properties(SculptGestureContext *sgcontext,
   trim_operation->mode = RNA_enum_get(op->ptr, "trim_mode");
   trim_operation->use_cursor_depth = RNA_boolean_get(op->ptr, "use_cursor_depth");
   trim_operation->orientation = RNA_enum_get(op->ptr, "trim_orientation");
+  trim_operation->location = RNA_enum_get(op->ptr, "trim_location");
 
   /* If the cursor was not over the mesh, force the orientation to view. */
   if (!sgcontext->ss->gesture_initial_hit) {
@@ -1389,6 +1420,13 @@ static void sculpt_trim_gesture_operator_properties(wmOperatorType *ot)
                SCULPT_GESTURE_TRIM_ORIENTATION_VIEW,
                "Shape Orientation",
                NULL);
+
+  RNA_def_enum(ot->srna,
+               "trim_location",
+               prop_trim_location_types,
+               SCULPT_GESTURE_TRIM_LOCATION_DEPTH_SURFACE,
+               "Shape Location",
+               NULL);
 }
 
 /* Project Gesture Operation. */
@@ -1582,7 +1620,7 @@ static int sculpt_trim_gesture_box_invoke(bContext *C, wmOperator *op, const wmE
   SculptCursorGeometryInfo sgi;
   float mouse[2] = {event->mval[0], event->mval[1]};
   SCULPT_vertex_random_access_ensure(ss);
-  ss->gesture_initial_hit = SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  ss->gesture_initial_hit = SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
   if (ss->gesture_initial_hit) {
     copy_v3_v3(ss->gesture_initial_location, sgi.location);
     copy_v3_v3(ss->gesture_initial_normal, sgi.normal);
@@ -1623,9 +1661,10 @@ static int sculpt_trim_gesture_lasso_invoke(bContext *C, wmOperator *op, const w
   SculptCursorGeometryInfo sgi;
   float mouse[2] = {event->mval[0], event->mval[1]};
   SCULPT_vertex_random_access_ensure(ss);
-  ss->gesture_initial_hit = SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  ss->gesture_initial_hit = SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, true);
   if (ss->gesture_initial_hit) {
     copy_v3_v3(ss->gesture_initial_location, sgi.location);
+    copy_v3_v3(ss->gesture_initial_back_location, sgi.back_location);
     copy_v3_v3(ss->gesture_initial_normal, sgi.normal);
   }
 
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index a53c2efed06..56502f932c2 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -2839,9 +2839,18 @@ typedef struct {
   const float *ray_start;
   const float *ray_normal;
   bool hit;
+  int hit_count;
+  bool back_hit;
   float depth;
   bool original;
 
+  /* Depth of the second raycast hit. */
+  float back_depth;
+
+  /* When the back depth is not needed, this can be set to false to avoid traversing unnecesary
+   * nodes. */
+  bool use_back_depth;
+
   int active_vertex_index;
   float *face_normal;
 
@@ -7257,8 +7266,8 @@ void SCULPT_stroke_modifiers_check(const bContext *C, Object *ob, const Brush *b
 
 static void sculpt_raycast_cb(PBVHNode *node, void *data_v, float *tmin)
 {
-  if (BKE_pbvh_node_get_tmin(node) < *tmin) {
-    SculptRaycastData *srd = data_v;
+  SculptRaycastData *srd = data_v;
+  if (srd->use_back_depth || BKE_pbvh_node_get_tmin(node) < *tmin) {
     float(*origco)[3] = NULL;
     bool use_origco = false;
 
@@ -7281,13 +7290,19 @@ static void sculpt_raycast_cb(PBVHNode *node, void *data_v, float *tmin)
                               srd->ray_start,
                               srd->ray_normal,
                               &srd->isect_precalc,
+                              &srd->hit_count,
                               &srd->depth,
+                              &srd->back_depth,
                               &srd->active_vertex_index,
                               &srd->active_face_grid_index,
                               srd->face_normal)) {
       srd->hit = true;
       *tmin = srd->depth;
     }
+
+    if (srd->hit_count >= 2) {
+      srd->back_hit = true;
+    }
   }
 }
 
@@ -7367,7 +7382,8 @@ float SCULPT_raycast_init(ViewContext *vc,
 bool SCULPT_cursor_geometry_info_update(bContext *C,
                                         SculptCursorGeometryInfo *out,
                                         const float mouse[2],
-                                        bool use_sampled_normal)
+                                        bool use_sampled_normal,
+                                        bool use_back_depth)
 {
   Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
   Scene *scene = CTX_data_scene(C);
@@ -7397,14 +7413,19 @@ bool SCULPT_cursor_geometry_info_update(bContext *C,
   /* PBVH raycast to get active vertex and face normal. */
   depth = SCULPT_raycast_init(&vc, mouse, ray_start, ray_end, ray_normal, original);
   SCULPT_stroke_modifiers_check(C, ob, brush);
+  float back_depth = depth;
 
   SculptRaycastData srd = {
       .original = original,
       .ss = ob->sculpt,
       .hit = false,
+      .back_hit = false,
       .ray_start = ray_start,
       .ray_normal = ray_normal,
       .depth = depth,
+      .back_depth = back_depth,
+      .hit_count = 0,
+      .use_back_depth = use_back_depth,
       .face_normal = face_normal,
   };
   isect_ray_tri_watertight_v3_precalc(&srd.isect_precalc, ray_normal);
@@ -7442,6 +7463,17 @@ bool SCULPT_cursor_geometry_info_update(bContext *C,
   mul_v3_fl(out->location, srd.depth);
   add_v3_v3(out->location, ray_start);
 
+  if (use_back_depth) {
+    copy_v3_v3(out->back_location, ray_normal);
+    if (srd.back_hit) {
+      mul_v3_fl(out->back_location, srd.back_depth);
+    }
+    else {
+      mul_v3_fl(out->back_location, srd.depth);
+    }
+    add_v3_v3(out->back_location, ray_start);
+  }
+
   /* Option to return the face normal directly for performance o accuracy reasons. */
   if (!use_sampled_normal) {
     copy_v3_v3(out->normal, srd.face_normal);
@@ -9272,7 +9304,7 @@ static int sculpt_mask_by_color_invoke(bContext *C, wmOperator *op, const wmEven
   float mouse[2];
   mouse[0] = event->mval[0];
   mouse[1] = event->mval[1];
-  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
 
   SCULPT_undo_push_begin(ob, "Mask by color");
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_cloth.c b/source/blender/editors/sculpt_paint/sculpt_cloth.c
index c97f31fa682..6246387bbdc 100644
--- a/source/blender/editors/sculpt_paint/sculpt_cloth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_cloth.c
@@ -1578,7 +1578,7 @@ static int sculpt_cloth_filter_invoke(bContext *C, wmOperator *op, const wmEvent
   SculptCursorGeometryInfo sgi;
   mouse[0] = event->mval[0];
   mouse[1] = event->mval[1];
-  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
 
   SCULPT_vertex_random_access_ensure(ss);
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_detail.c b/source/blender/editors/sculpt_paint/sculpt_detail.c
index aa1d407dc24..fcf281cb44d 100644
--- a/source/blender/editors/sculpt_paint/sculpt_detail.c
+++ b/source/blender/editors/sculpt_paint/sculpt_detail.c
@@ -180,7 +180,7 @@ static void sample_detail_voxel(bContext *C, ViewContext *vc, int mx, int my)
 
   /* Update the active vertex. */
   const float mouse[2] = {mx, my};
-  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
   BKE_sculpt_update_object_for_edit(depsgraph, ob, true, false, false);
 
   /* Average the edge length of the connected edges to the active vertex. */
diff --git a/source/blender/editors/sculpt_paint/sculpt_face_set.c b/source/blender/editors/sculpt_paint/sculpt_face_set.c
index 20d8e136d12..486d318f8b6 100644
--- a/source/blender/editors/sculpt_paint/sculpt_face_set.c
+++ b/source/blender/editors/sculpt_paint/sculpt_face_set.c
@@ -111,7 +111,7 @@ int ED_sculpt_face_sets_active_update_and_get(bContext *C, Object *ob, const flo
   }
 
   SculptCursorGeometryInfo gi;
-  if (!SCULPT_cursor_geometry_info_update(C, &gi, mval, false)) {
+  if (!SCULPT_cursor_geometry_info_update(C, &gi, mval, false, false)) {
     return SCULPT_FACE_SET_NONE;
   }
 
@@ -943,7 +943,7 @@ static int sculpt_face_sets_change_visibility_invoke(bContext *C,
   mouse[0] = event->mval[0];
   mouse[1] = event->mval[1];
   SCULPT_vertex_random_access_ensure(ss);
-  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
 
   return sculpt_face_sets_change_visibility_exec(C, op);
 }
@@ -1391,10 +1391,12 @@ static int sculpt_face_set_edit_invoke(bContext *C, wmOperator *op, const wmEven
    * tool without brush cursor. */
   SculptCursorGeometryInfo sgi;
   const float mouse[2] = {event->mval[0], event->mval[1]};
-  if (!SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false)) {
+
+  if (!SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false)) {
     /* The cursor is not over the mesh. Cancel to avoid editing the last updated Face Set ID. */
     return OPERATOR_CANCELLED;
   }
+
   const int active_face_set = SCULPT_active_face_set_get(ss);
 
   switch (mode) {
diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_color.c b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
index 76a6b05cdff..f2d7613eefb 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_color.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
@@ -275,7 +275,7 @@ static int sculpt_color_filter_invoke(bContext *C, wmOperator *op, const wmEvent
     SculptCursorGeometryInfo sgi;
     mouse[0] = event->mval[0];
     mouse[1] = event->mval[1];
-    SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+    SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
   }
 
   /* Disable for multires and dyntopo for now */
diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
index 3cf6a8cc561..65b2cdb9df2 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
@@ -694,7 +694,7 @@ static int sculpt_mesh_filter_invoke(bContext *C, wmOperator *op, const wmEvent
     SculptCursorGeometryInfo sgi;
     mouse[0] = event->mval[0];
     mouse[1] = event->mval[1];
-    SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+    SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
   }
 
   SCULPT_vertex_random_access_ensure(ss);
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index f90cf366ed9..d8bbd4aa190 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -72,6 +72,7 @@ void SCULPT_tag_update_overlays(bContext *C);
 
 typedef struct SculptCursorGeometryInfo {
   float location[3];
+  float back_location[3];
   float normal[3];
   float active_vertex_co[3];
 } SculptCursorGeometryInfo;
@@ -80,7 +81,8 @@ bool SCULPT_stroke_get_location(struct bContext *C, float out[3], const float mo
 bool SCULPT_cursor_geometry_info_update(bContext *C,
                                         SculptCursorGeometryInfo *out,
                                         const float mouse[2],
-                                        bool use_sampled_normal);
+                                        bool use_sampled_normal,
+                                        bool use_back_depth);
 void SCULPT_geometry_preview_lines_update(bContext *C, struct SculptSession *ss, float radius);
 
 void SCULPT_stroke_modifiers_check(const bContext *C, Object *ob, const Brush *brush);
diff --git a/source/blender/editors/sculpt_paint/sculpt_mask_expand.c b/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
index 5e229e020ad..7a966568076 100644
--- a/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
+++ b/source/blender/editors/sculpt_paint/sculpt_mask_expand.c
@@ -188,7 +188,7 @@ static int sculpt_mask_expand_modal(bContext *C, wmOperator *op, const wmEvent *
     float mouse[2];
     mouse[0] = event->mval[0];
     mouse[1] = event->mval[1];
-    if (SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false)) {
+    if (SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false)) {
       /* The cursor is over the mesh, get the update iteration from the updated active vertex. */
       mask_expand_update_it = ss->filter_cache->mask_update_it[(int)SCULPT_active_vertex_get(ss)];
     }
@@ -368,7 +368,7 @@ static int sculpt_mask_expand_invoke(bContext *C, wmOperator *op, const wmEvent
   op->customdata = MEM_mallocN(sizeof(float[2]), "initial mouse position");
   copy_v2_v2(op->customdata, mouse);
 
-  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false);
+  SCULPT_cursor_geometry_info_update(C, &sgi, mouse, false, false);
 
   BKE_sculpt_update_object_for_edit(depsgraph, ob, true, true, false);