Sculpt: fix T101465, crash in cloth filter with new automasking modes

4 files changed, 19 insertions, 3 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index ed7ef5d5efd..5312292d431 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -372,6 +372,7 @@ typedef struct SculptClothSimulation {
   float (*acceleration)[3];
   float (*pos)[3];
   float (*init_pos)[3];
+  float (*init_no)[3];
   float (*softbody_pos)[3];
   float (*prev_pos)[3];
   float (*last_iteration_pos)[3];
diff --git a/source/blender/editors/sculpt_paint/sculpt_cloth.c b/source/blender/editors/sculpt_paint/sculpt_cloth.c
index 2b365a661ee..36093228f7d 100644
--- a/source/blender/editors/sculpt_paint/sculpt_cloth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_cloth.c
@@ -509,6 +509,8 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
       data->ob, ss, SCULPT_automasking_active_cache_get(ss), &automask_data, data->nodes[n]);
 
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float force[3];
     float sim_location[3];
     cloth_brush_simulation_location_get(ss, brush, sim_location);
@@ -775,6 +777,8 @@ static void do_cloth_brush_solve_simulation_task_cb_ex(
       data->ob, ss, SCULPT_automasking_active_cache_get(ss), &automask_data, data->nodes[n]);
 
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float sim_location[3];
     cloth_brush_simulation_location_get(ss, brush, sim_location);
     const float sim_factor =
@@ -870,9 +874,14 @@ static void cloth_brush_satisfy_constraints(SculptSession *ss,
       PBVHVertRef vertex1 = BKE_pbvh_index_to_vertex(ss->pbvh, v1);
       PBVHVertRef vertex2 = BKE_pbvh_index_to_vertex(ss->pbvh, v2);
 
+      automask_data.orig_data.co = cloth_sim->init_pos[v1];
+      automask_data.orig_data.no = cloth_sim->init_no[v1];
       const float mask_v1 = (1.0f - SCULPT_vertex_mask_get(ss, vertex1)) *
                             SCULPT_automasking_factor_get(
                                 automasking, ss, vertex1, &automask_data);
+
+      automask_data.orig_data.co = cloth_sim->init_pos[v2];
+      automask_data.orig_data.no = cloth_sim->init_no[v2];
       const float mask_v2 = (1.0f - SCULPT_vertex_mask_get(ss, vertex2)) *
                             SCULPT_automasking_factor_get(
                                 automasking, ss, vertex2, &automask_data);
@@ -1078,6 +1087,7 @@ SculptClothSimulation *SCULPT_cloth_brush_simulation_create(Object *ob,
   cloth_sim->last_iteration_pos = MEM_calloc_arrayN(
       totverts, sizeof(float[3]), "cloth sim last iteration pos");
   cloth_sim->init_pos = MEM_calloc_arrayN(totverts, sizeof(float[3]), "cloth sim init pos");
+  cloth_sim->init_no = MEM_calloc_arrayN(totverts, sizeof(float[3]), "cloth sim init normals");
   cloth_sim->length_constraint_tweak = MEM_calloc_arrayN(
       totverts, sizeof(float), "cloth sim length tweak");
 
@@ -1153,6 +1163,7 @@ void SCULPT_cloth_brush_simulation_init(SculptSession *ss, SculptClothSimulation
 
     copy_v3_v3(cloth_sim->last_iteration_pos[i], SCULPT_vertex_co_get(ss, vertex));
     copy_v3_v3(cloth_sim->init_pos[i], SCULPT_vertex_co_get(ss, vertex));
+    SCULPT_vertex_normal_get(ss, vertex, cloth_sim->init_no[i]);
     copy_v3_v3(cloth_sim->prev_pos[i], SCULPT_vertex_co_get(ss, vertex));
     if (has_deformation_pos) {
       copy_v3_v3(cloth_sim->deformation_pos[i], SCULPT_vertex_co_get(ss, vertex));
@@ -1266,6 +1277,7 @@ void SCULPT_cloth_simulation_free(struct SculptClothSimulation *cloth_sim)
   MEM_SAFE_FREE(cloth_sim->deformation_pos);
   MEM_SAFE_FREE(cloth_sim->softbody_pos);
   MEM_SAFE_FREE(cloth_sim->init_pos);
+  MEM_SAFE_FREE(cloth_sim->init_no);
   MEM_SAFE_FREE(cloth_sim->deformation_strength);
   MEM_SAFE_FREE(cloth_sim->node_state);
   BLI_ghash_free(cloth_sim->node_state_index, NULL, NULL);
@@ -1453,6 +1465,8 @@ static void cloth_filter_apply_forces_task_cb(void *__restrict userdata,
 
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float fade = vd.mask ? *vd.mask : 0.0f;
     fade *= SCULPT_automasking_factor_get(
         ss->filter_cache->automasking, ss, vd.vertex, &automask_data);
@@ -1597,6 +1611,8 @@ static int sculpt_cloth_filter_invoke(bContext *C, wmOperator *op, const wmEvent
   /* Needs mask data to be available as it is used when solving the constraints. */
   BKE_sculpt_update_object_for_edit(depsgraph, ob, true, true, false);
 
+  SCULPT_stroke_id_next(ob);
+
   SCULPT_undo_push_begin(ob, op);
   SCULPT_filter_cache_init(
       C, ob, sd, SCULPT_UNDO_COORDS, event->mval, RNA_float_get(op->ptr, "area_normal_radius"));
diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_color.c b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
index 42ea02caff8..0d21aa24e7d 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_color.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
@@ -102,6 +102,8 @@ static void color_filter_task_cb(void *__restrict userdata,
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
     SCULPT_orig_vert_data_update(&orig_data, &vd);
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float orig_color[3], final_color[4], hsv_color[3];
     int hue;
     float brightness, contrast, gain, delta, offset;
@@ -295,8 +297,6 @@ static int sculpt_color_filter_modal(bContext *C, wmOperator *op, const wmEvent
     return OPERATOR_RUNNING_MODAL;
   }
 
-  SCULPT_stroke_id_next(ob);
-
   const float len = event->prev_press_xy[0] - event->xy[0];
   filter_strength = filter_strength * -len * 0.001f;
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
index 69a51d22210..3fdc2eaff0f 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
@@ -712,7 +712,6 @@ static int sculpt_mesh_filter_modal(bContext *C, wmOperator *op, const wmEvent *
   const float len = event->prev_press_xy[0] - event->xy[0];
   filter_strength = filter_strength * -len * 0.001f * UI_DPI_FAC;
 
-  SCULPT_stroke_id_next(ob);
   SCULPT_vertex_random_access_ensure(ss);
 
   bool needs_pmap = sculpt_mesh_filter_needs_pmap(filter_type);