Fix Cloth Brush grab deformation mode

The grab mode was not correctly implemented, so the way it was working
was confusing for users.
- Grab delta was calculated in increments from the last stroke position, so it did not match the behavior of a grab brush. I refactored the grab delta calculation to make this change more explicit.
- Grab displacement was not calculated from the original coordinates
- Grab was using an incorrect strength

Grab is now setting the position of the affected vertices directly and
the constraints solve the rest of the cloth. I also tried to implement
an alternative version based on applying forces to move the vertices to
the grab position, but I think this is more controllable and the grab
falloff can be adjusted by tweaking the simulation falloff.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D7756

4 files changed, 90 insertions, 50 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_stroke.c b/source/blender/editors/sculpt_paint/paint_stroke.c
index 7887aaaf658..463334720c5 100644
--- a/source/blender/editors/sculpt_paint/paint_stroke.c
+++ b/source/blender/editors/sculpt_paint/paint_stroke.c
@@ -230,8 +230,7 @@ static bool paint_tool_require_location(Brush *brush, ePaintMode mode)
                SCULPT_TOOL_THUMB)) {
         return false;
       }
-      else if (brush->sculpt_tool == SCULPT_TOOL_CLOTH &&
-               brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_GRAB) {
+      else if (SCULPT_is_cloth_deform_brush(brush)) {
         return false;
       }
       else {
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 8443608cfa5..c42170695c2 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -2091,9 +2091,13 @@ static float brush_strength(const Sculpt *sd,
     case SCULPT_TOOL_LAYER:
       return alpha * flip * pressure * overlap * feather;
     case SCULPT_TOOL_CLOTH:
-      /* Expand is more sensible to strength as it keeps expanding the cloth when sculpting over
-       * the same vertices. */
-      if (brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_EXPAND) {
+      if (brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_GRAB) {
+        /* Grab deform uses the same falloff as a regular grab brush. */
+        return root_alpha * feather;
+      }
+      else if (brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_EXPAND) {
+        /* Expand is more sensible to strength as it keeps expanding the cloth when sculpting over
+         * the same vertices. */
         return 0.1f * alpha * flip * pressure * overlap * feather;
       }
       else {
@@ -6199,6 +6203,34 @@ static float sculpt_brush_dynamic_size_get(Brush *brush, StrokeCache *cache, flo
   }
 }
 
+/* In these brushes the grab delta is calculated always from the initial stroke location, which is
+ * generally used to create grab deformations. */
+static bool sculpt_needs_delta_from_anchored_origin(Brush *brush)
+{
+  return ELEM(brush->sculpt_tool,
+              SCULPT_TOOL_GRAB,
+              SCULPT_TOOL_POSE,
+              SCULPT_TOOL_THUMB,
+              SCULPT_TOOL_ELASTIC_DEFORM) ||
+         SCULPT_is_cloth_deform_brush(brush);
+}
+
+/* In these brushes the grab delta is calculated from the previous stroke location, which is used
+ * to calculate to orientate the brush tip and deformation towards the stroke direction. */
+static bool sculpt_needs_delta_for_tip_orientation(Brush *brush)
+{
+  if (brush->sculpt_tool == SCULPT_TOOL_CLOTH) {
+    return !SCULPT_is_cloth_deform_brush(brush);
+  }
+  return ELEM(brush->sculpt_tool,
+              SCULPT_TOOL_CLAY_STRIPS,
+              SCULPT_TOOL_PINCH,
+              SCULPT_TOOL_MULTIPLANE_SCRAPE,
+              SCULPT_TOOL_CLAY_THUMB,
+              SCULPT_TOOL_NUDGE,
+              SCULPT_TOOL_SNAKE_HOOK);
+}
+
 static void sculpt_update_brush_delta(UnifiedPaintSettings *ups, Object *ob, Brush *brush)
 {
   SculptSession *ss = ob->sculpt;
@@ -6242,38 +6274,27 @@ static void sculpt_update_brush_delta(UnifiedPaintSettings *ups, Object *ob, Bru
 
     /* Compute delta to move verts by. */
     if (!cache->first_time) {
-      switch (tool) {
-        case SCULPT_TOOL_GRAB:
-        case SCULPT_TOOL_POSE:
-        case SCULPT_TOOL_THUMB:
-        case SCULPT_TOOL_ELASTIC_DEFORM:
-          sub_v3_v3v3(delta, grab_location, cache->old_grab_location);
-          invert_m4_m4(imat, ob->obmat);
-          mul_mat3_m4_v3(imat, delta);
-          add_v3_v3(cache->grab_delta, delta);
-          break;
-        case SCULPT_TOOL_CLAY_STRIPS:
-        case SCULPT_TOOL_PINCH:
-        case SCULPT_TOOL_CLOTH:
-        case SCULPT_TOOL_MULTIPLANE_SCRAPE:
-        case SCULPT_TOOL_CLAY_THUMB:
-        case SCULPT_TOOL_NUDGE:
-        case SCULPT_TOOL_SNAKE_HOOK:
-          if (brush->flag & BRUSH_ANCHORED) {
-            float orig[3];
-            mul_v3_m4v3(orig, ob->obmat, cache->orig_grab_location);
-            sub_v3_v3v3(cache->grab_delta, grab_location, orig);
-          }
-          else {
-            sub_v3_v3v3(cache->grab_delta, grab_location, cache->old_grab_location);
-          }
-          invert_m4_m4(imat, ob->obmat);
-          mul_mat3_m4_v3(imat, cache->grab_delta);
-          break;
-        default:
-          /* Use for 'Brush.topology_rake_factor'. */
+      if (sculpt_needs_delta_from_anchored_origin(brush)) {
+        sub_v3_v3v3(delta, grab_location, cache->old_grab_location);
+        invert_m4_m4(imat, ob->obmat);
+        mul_mat3_m4_v3(imat, delta);
+        add_v3_v3(cache->grab_delta, delta);
+      }
+      else if (sculpt_needs_delta_for_tip_orientation(brush)) {
+        if (brush->flag & BRUSH_ANCHORED) {
+          float orig[3];
+          mul_v3_m4v3(orig, ob->obmat, cache->orig_grab_location);
+          sub_v3_v3v3(cache->grab_delta, grab_location, orig);
+        }
+        else {
           sub_v3_v3v3(cache->grab_delta, grab_location, cache->old_grab_location);
-          break;
+        }
+        invert_m4_m4(imat, ob->obmat);
+        mul_mat3_m4_v3(imat, cache->grab_delta);
+      }
+      else {
+        /* Use for 'Brush.topology_rake_factor'. */
+        sub_v3_v3v3(cache->grab_delta, grab_location, cache->old_grab_location);
       }
     }
     else {
@@ -6294,18 +6315,14 @@ static void sculpt_update_brush_delta(UnifiedPaintSettings *ups, Object *ob, Bru
         copy_v3_v3(cache->anchored_location, cache->true_location);
       }
     }
-    else if (tool == SCULPT_TOOL_ELASTIC_DEFORM) {
+    else if (tool == SCULPT_TOOL_ELASTIC_DEFORM || SCULPT_is_cloth_deform_brush(brush)) {
       copy_v3_v3(cache->anchored_location, cache->true_location);
     }
     else if (tool == SCULPT_TOOL_THUMB) {
       copy_v3_v3(cache->anchored_location, cache->orig_grab_location);
     }
 
-    if (ELEM(tool,
-             SCULPT_TOOL_GRAB,
-             SCULPT_TOOL_THUMB,
-             SCULPT_TOOL_ELASTIC_DEFORM,
-             SCULPT_TOOL_POSE)) {
+    if (sculpt_needs_delta_from_anchored_origin(brush)) {
       /* Location stays the same for finding vertices in brush radius. */
       copy_v3_v3(cache->true_location, cache->orig_grab_location);
 
@@ -6376,9 +6393,7 @@ static void sculpt_update_cache_variants(bContext *C, Sculpt *sd, Object *ob, Po
 
   if (cache->first_time ||
       !((brush->flag & BRUSH_ANCHORED) || (brush->sculpt_tool == SCULPT_TOOL_SNAKE_HOOK) ||
-        (brush->sculpt_tool == SCULPT_TOOL_ROTATE) ||
-        (brush->sculpt_tool == SCULPT_TOOL_CLOTH &&
-         brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_GRAB))) {
+        (brush->sculpt_tool == SCULPT_TOOL_ROTATE) || SCULPT_is_cloth_deform_brush(brush))) {
     RNA_float_get_array(ptr, "location", cache->true_location);
   }
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_cloth.c b/source/blender/editors/sculpt_paint/sculpt_cloth.c
index b1cc6d02bbb..84d575c92d1 100644
--- a/source/blender/editors/sculpt_paint/sculpt_cloth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_cloth.c
@@ -212,8 +212,9 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
   const float *grab_delta = data->grab_delta;
   float(*imat)[4] = data->mat;
 
-  const bool use_falloff_plane = brush->cloth_force_falloff_type ==
-                                 BRUSH_CLOTH_FORCE_FALLOFF_PLANE;
+  const bool use_falloff_plane = !SCULPT_is_cloth_deform_brush(brush) &&
+                                 brush->cloth_force_falloff_type ==
+                                     BRUSH_CLOTH_FORCE_FALLOFF_PLANE;
 
   PBVHVertexIter vd;
   const float bstrength = ss->cache->bstrength;
@@ -245,14 +246,29 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
         gravity, ss->cache->gravity_direction, -ss->cache->radius * data->sd->gravity_factor);
   }
 
+  /* Original data for deform brushes. */
+  SculptOrigVertData orig_data;
+  if (SCULPT_is_cloth_deform_brush(brush)) {
+    SCULPT_orig_vert_data_init(&orig_data, data->ob, data->nodes[n]);
+  }
+
   BKE_pbvh_vertex_iter_begin(ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE)
   {
     float force[3];
     const float sim_factor = cloth_brush_simulation_falloff_get(
         brush, ss->cache->radius, ss->cache->initial_location, cloth_sim->init_pos[vd.index]);
 
+    float current_vertex_location[3];
+    if (SCULPT_is_cloth_deform_brush(brush)) {
+      SCULPT_orig_vert_data_update(&orig_data, &vd);
+      copy_v3_v3(current_vertex_location, orig_data.co);
+    }
+    else {
+      copy_v3_v3(current_vertex_location, vd.co);
+    }
+
     /* When using the plane falloff mode the falloff is not constrained by the brush radius. */
-    if (sculpt_brush_test_sq_fn(&test, vd.co) || use_falloff_plane) {
+    if (sculpt_brush_test_sq_fn(&test, current_vertex_location) || use_falloff_plane) {
 
       float dist = sqrtf(test.dist);
 
@@ -263,7 +279,7 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
       const float fade = sim_factor * bstrength *
                          SCULPT_brush_strength_factor(ss,
                                                       brush,
-                                                      vd.co,
+                                                      current_vertex_location,
                                                       dist,
                                                       vd.no,
                                                       vd.fno,
@@ -292,7 +308,10 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
           mul_v3_v3fl(force, offset, -fade);
           break;
         case BRUSH_CLOTH_DEFORM_GRAB:
-          mul_v3_v3fl(force, grab_delta, fade);
+          /* Grab writes the positions in the simulation directly without applying forces. */
+          madd_v3_v3v3fl(
+              cloth_sim->pos[vd.index], orig_data.co, ss->cache->grab_delta_symmetry, fade);
+          zero_v3(force);
           break;
         case BRUSH_CLOTH_DEFORM_PINCH_POINT:
           if (use_falloff_plane) {
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index 191a6fdd62e..ec7ea1d85f4 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -333,6 +333,13 @@ void SCULPT_cloth_plane_falloff_preview_draw(const uint gpuattr,
                                              struct SculptSession *ss,
                                              const float outline_col[3],
                                              float outline_alpha);
+
+BLI_INLINE bool SCULPT_is_cloth_deform_brush(const Brush *brush)
+{
+  return brush->sculpt_tool == SCULPT_TOOL_CLOTH &&
+         brush->cloth_deform_type == BRUSH_CLOTH_DEFORM_GRAB;
+}
+
 /* Pose Brush. */
 void SCULPT_do_pose_brush(struct Sculpt *sd,
                           struct Object *ob,