Sculpt: Refactor transform code to allow incremental updates

This adds support for incremental updates in the sculpt transform
code. Now tools can define if they need the displacement applied
for the original coordinates or incrementally.

This is needed for features like elastic transform or cloth deformation
target in the transform tool.

No functional changes.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D9547

3 files changed, 119 insertions, 53 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index c33dfecd52c..7d5f71b070f 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -2632,7 +2632,8 @@ void SCULPT_clip(Sculpt *sd, SculptSession *ss, float co[3], const float val[3])
       continue;
     }
 
-    if ((ss->cache->flag & (CLIP_X << i)) && (fabsf(co[i]) <= ss->cache->clip_tolerance[i])) {
+    if (ss->cache && (ss->cache->flag & (CLIP_X << i)) &&
+        (fabsf(co[i]) <= ss->cache->clip_tolerance[i])) {
       co[i] = 0.0f;
     }
     else {
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index c5db078617f..99ee22328ea 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -1015,6 +1015,14 @@ typedef enum SculptFilterOrientation {
   SCULPT_FILTER_ORIENTATION_VIEW = 2,
 } SculptFilterOrientation;
 
+/* Defines how transform tools are going to apply its displacement. */
+typedef enum SculptTransformDisplacementMode {
+  /* Displaces the elements from their original coordinates. */
+  SCULPT_TRANSFORM_DISPLACEMENT_ORIGINAL = 0,
+  /* Displaces the elements incrementally from their previous position. */
+  SCULPT_TRANSFORM_DISPLACEMENT_INCREMENTAL = 1,
+} SculptTransformDisplacementMode;
+
 void SCULPT_filter_to_orientation_space(float r_v[3], struct FilterCache *filter_cache);
 void SCULPT_filter_to_object_space(float r_v[3], struct FilterCache *filter_cache);
 void SCULPT_filter_zero_disabled_axis_components(float r_v[3], struct FilterCache *filter_cache);
@@ -1072,6 +1080,9 @@ typedef struct FilterCache {
 
   int active_face_set;
 
+  /* Transform. */
+  SculptTransformDisplacementMode transform_displacement_mode;
+
   /* Auto-masking. */
   AutomaskingCache *automasking;
 } FilterCache;
diff --git a/source/blender/editors/sculpt_paint/sculpt_transform.c b/source/blender/editors/sculpt_paint/sculpt_transform.c
index c1281c98deb..88585745467 100644
--- a/source/blender/editors/sculpt_paint/sculpt_transform.c
+++ b/source/blender/editors/sculpt_paint/sculpt_transform.c
@@ -69,6 +69,11 @@ void ED_sculpt_init_transform(struct bContext *C)
 
   copy_v3_v3(ss->init_pivot_pos, ss->pivot_pos);
   copy_v4_v4(ss->init_pivot_rot, ss->pivot_rot);
+  copy_v3_v3(ss->init_pivot_scale, ss->pivot_scale);
+
+  copy_v3_v3(ss->prev_pivot_pos, ss->pivot_pos);
+  copy_v4_v4(ss->prev_pivot_rot, ss->pivot_rot);
+  copy_v3_v3(ss->prev_pivot_scale, ss->pivot_scale);
 
   SCULPT_undo_push_begin(ob, "Transform");
   BKE_sculpt_update_object_for_edit(depsgraph, ob, false, false, false);
@@ -77,6 +82,72 @@ void ED_sculpt_init_transform(struct bContext *C)
 
   SCULPT_vertex_random_access_ensure(ss);
   SCULPT_filter_cache_init(C, ob, sd, SCULPT_UNDO_COORDS);
+
+  ss->filter_cache->transform_displacement_mode = SCULPT_TRANSFORM_DISPLACEMENT_ORIGINAL;
+}
+
+static void sculpt_transform_matrices_init(SculptSession *ss,
+                                           const char symm,
+                                           const SculptTransformDisplacementMode t_mode,
+                                           float r_transform_mats[8][4][4])
+{
+
+  float final_pivot_pos[3], d_t[3], d_r[4], d_s[3];
+  float t_mat[4][4], r_mat[4][4], s_mat[4][4], pivot_mat[4][4], pivot_imat[4][4],
+      transform_mat[4][4];
+
+  float start_pivot_pos[3], start_pivot_rot[4], start_pivot_scale[3];
+  switch (t_mode) {
+    case SCULPT_TRANSFORM_DISPLACEMENT_ORIGINAL:
+      copy_v3_v3(start_pivot_pos, ss->init_pivot_pos);
+      copy_v4_v4(start_pivot_rot, ss->init_pivot_rot);
+      copy_v3_v3(start_pivot_scale, ss->init_pivot_scale);
+      break;
+    case SCULPT_TRANSFORM_DISPLACEMENT_INCREMENTAL:
+      copy_v3_v3(start_pivot_pos, ss->prev_pivot_pos);
+      copy_v4_v4(start_pivot_rot, ss->prev_pivot_rot);
+      copy_v3_v3(start_pivot_scale, ss->prev_pivot_scale);
+      break;
+  }
+
+  for (int i = 0; i < PAINT_SYMM_AREAS; i++) {
+    ePaintSymmetryAreas v_symm = i;
+
+    copy_v3_v3(final_pivot_pos, ss->pivot_pos);
+
+    unit_m4(pivot_mat);
+
+    unit_m4(t_mat);
+    unit_m4(r_mat);
+    unit_m4(s_mat);
+
+    /* Translation matrix. */
+    sub_v3_v3v3(d_t, ss->pivot_pos, start_pivot_pos);
+    SCULPT_flip_v3_by_symm_area(d_t, symm, v_symm, ss->init_pivot_pos);
+    translate_m4(t_mat, d_t[0], d_t[1], d_t[2]);
+
+    /* Rotation matrix. */
+    sub_qt_qtqt(d_r, ss->pivot_rot, start_pivot_rot);
+    normalize_qt(d_r);
+    SCULPT_flip_quat_by_symm_area(d_r, symm, v_symm, ss->init_pivot_pos);
+    quat_to_mat4(r_mat, d_r);
+
+    /* Scale matrix. */
+    sub_v3_v3v3(d_s, ss->pivot_scale, start_pivot_scale);
+    add_v3_fl(d_s, 1.0f);
+    size_to_mat4(s_mat, d_s);
+
+    /* Pivot matrix. */
+    SCULPT_flip_v3_by_symm_area(final_pivot_pos, symm, v_symm, start_pivot_pos);
+    translate_m4(pivot_mat, final_pivot_pos[0], final_pivot_pos[1], final_pivot_pos[2]);
+    invert_m4_m4(pivot_imat, pivot_mat);
+
+    /* Final transform matrix. */
+    mul_m4_m4m4(transform_mat, r_mat, t_mat);
+    mul_m4_m4m4(transform_mat, transform_mat, s_mat);
+    mul_m4_m4m4(r_transform_mats[i], transform_mat, pivot_imat);
+    mul_m4_m4m4(r_transform_mats[i], pivot_mat, r_transform_mats[i]);
+  }
 }
 
 static void sculpt_transform_task_cb(void *__restrict userdata,
@@ -98,16 +169,25 @@ static void sculpt_transform_task_cb(void *__restrict userdata,
   {
     SCULPT_orig_vert_data_update(&orig_data, &vd);
     float transformed_co[3], orig_co[3], disp[3];
+    float *start_co;
     float fade = vd.mask ? *vd.mask : 0.0f;
     copy_v3_v3(orig_co, orig_data.co);
     char symm_area = SCULPT_get_vertex_symm_area(orig_co);
 
-    copy_v3_v3(transformed_co, orig_co);
+    switch (ss->filter_cache->transform_displacement_mode) {
+      case SCULPT_TRANSFORM_DISPLACEMENT_ORIGINAL:
+        start_co = orig_co;
+        break;
+      case SCULPT_TRANSFORM_DISPLACEMENT_INCREMENTAL:
+        start_co = vd.co;
+        break;
+    }
+
+    copy_v3_v3(transformed_co, start_co);
     mul_m4_v3(data->transform_mats[(int)symm_area], transformed_co);
-    sub_v3_v3v3(disp, transformed_co, orig_co);
+    sub_v3_v3v3(disp, transformed_co, start_co);
     mul_v3_fl(disp, 1.0f - fade);
-
-    add_v3_v3v3(vd.co, orig_co, disp);
+    add_v3_v3v3(vd.co, start_co, disp);
 
     if (vd.mvert) {
       vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
@@ -118,69 +198,43 @@ static void sculpt_transform_task_cb(void *__restrict userdata,
   BKE_pbvh_node_mark_update(node);
 }
 
-void ED_sculpt_update_modal_transform(struct bContext *C)
+static void sculpt_transform_all_vertices(Sculpt *sd, Object *ob)
 {
-  Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
-  Object *ob = CTX_data_active_object(C);
   SculptSession *ss = ob->sculpt;
-  Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
   const char symm = SCULPT_mesh_symmetry_xyz_get(ob);
 
-  SCULPT_vertex_random_access_ensure(ss);
-  BKE_sculpt_update_object_for_edit(depsgraph, ob, false, false, false);
-
   SculptThreadedTaskData data = {
       .sd = sd,
       .ob = ob,
       .nodes = ss->filter_cache->nodes,
   };
 
-  float final_pivot_pos[3], d_t[3], d_r[4];
-  float t_mat[4][4], r_mat[4][4], s_mat[4][4], pivot_mat[4][4], pivot_imat[4][4],
-      transform_mat[4][4];
-
-  copy_v3_v3(final_pivot_pos, ss->pivot_pos);
-  for (int i = 0; i < PAINT_SYMM_AREAS; i++) {
-    ePaintSymmetryAreas v_symm = i;
-
-    copy_v3_v3(final_pivot_pos, ss->pivot_pos);
-
-    unit_m4(pivot_mat);
-
-    unit_m4(t_mat);
-    unit_m4(r_mat);
-    unit_m4(s_mat);
-
-    /* Translation matrix. */
-    sub_v3_v3v3(d_t, ss->pivot_pos, ss->init_pivot_pos);
-    SCULPT_flip_v3_by_symm_area(d_t, symm, v_symm, ss->init_pivot_pos);
-    translate_m4(t_mat, d_t[0], d_t[1], d_t[2]);
-
-    /* Rotation matrix. */
-    sub_qt_qtqt(d_r, ss->pivot_rot, ss->init_pivot_rot);
-    normalize_qt(d_r);
-    SCULPT_flip_quat_by_symm_area(d_r, symm, v_symm, ss->init_pivot_pos);
-    quat_to_mat4(r_mat, d_r);
-
-    /* Scale matrix. */
-    size_to_mat4(s_mat, ss->pivot_scale);
-
-    /* Pivot matrix. */
-    SCULPT_flip_v3_by_symm_area(final_pivot_pos, symm, v_symm, ss->init_pivot_pos);
-    translate_m4(pivot_mat, final_pivot_pos[0], final_pivot_pos[1], final_pivot_pos[2]);
-    invert_m4_m4(pivot_imat, pivot_mat);
-
-    /* Final transform matrix. */
-    mul_m4_m4m4(transform_mat, r_mat, t_mat);
-    mul_m4_m4m4(transform_mat, transform_mat, s_mat);
-    mul_m4_m4m4(data.transform_mats[i], transform_mat, pivot_imat);
-    mul_m4_m4m4(data.transform_mats[i], pivot_mat, data.transform_mats[i]);
-  }
+  sculpt_transform_matrices_init(
+      ss, symm, ss->filter_cache->transform_displacement_mode, data.transform_mats);
 
+  /* Regular transform applies all symmetry passes at once as it is split by symmetry areas (each
+   * vertex can only be transformed once by the transform matix of its area). */
   TaskParallelSettings settings;
   BKE_pbvh_parallel_range_settings(&settings, true, ss->filter_cache->totnode);
   BLI_task_parallel_range(
       0, ss->filter_cache->totnode, &data, sculpt_transform_task_cb, &settings);
+}
+
+void ED_sculpt_update_modal_transform(struct bContext *C)
+{
+  Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
+  Object *ob = CTX_data_active_object(C);
+  SculptSession *ss = ob->sculpt;
+  Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C);
+
+  SCULPT_vertex_random_access_ensure(ss);
+  BKE_sculpt_update_object_for_edit(depsgraph, ob, false, false, false);
+
+  sculpt_transform_all_vertices(sd, ob);
+
+  copy_v3_v3(ss->prev_pivot_pos, ss->pivot_pos);
+  copy_v4_v4(ss->prev_pivot_rot, ss->pivot_rot);
+  copy_v3_v3(ss->prev_pivot_scale, ss->pivot_scale);
 
   if (ss->deform_modifiers_active || ss->shapekey_active) {
     SCULPT_flush_stroke_deform(sd, ob, true);