Sculpt: fix T101465, crash in cloth filter with new automasking modes

1 files changed, 16 insertions, 0 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_cloth.c b/source/blender/editors/sculpt_paint/sculpt_cloth.c
index 2b365a661ee..36093228f7d 100644
--- a/source/blender/editors/sculpt_paint/sculpt_cloth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_cloth.c
@@ -509,6 +509,8 @@ static void do_cloth_brush_apply_forces_task_cb_ex(void *__restrict userdata,
       data->ob, ss, SCULPT_automasking_active_cache_get(ss), &automask_data, data->nodes[n]);
 
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float force[3];
     float sim_location[3];
     cloth_brush_simulation_location_get(ss, brush, sim_location);
@@ -775,6 +777,8 @@ static void do_cloth_brush_solve_simulation_task_cb_ex(
       data->ob, ss, SCULPT_automasking_active_cache_get(ss), &automask_data, data->nodes[n]);
 
   BKE_pbvh_vertex_iter_begin (ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float sim_location[3];
     cloth_brush_simulation_location_get(ss, brush, sim_location);
     const float sim_factor =
@@ -870,9 +874,14 @@ static void cloth_brush_satisfy_constraints(SculptSession *ss,
       PBVHVertRef vertex1 = BKE_pbvh_index_to_vertex(ss->pbvh, v1);
       PBVHVertRef vertex2 = BKE_pbvh_index_to_vertex(ss->pbvh, v2);
 
+      automask_data.orig_data.co = cloth_sim->init_pos[v1];
+      automask_data.orig_data.no = cloth_sim->init_no[v1];
       const float mask_v1 = (1.0f - SCULPT_vertex_mask_get(ss, vertex1)) *
                             SCULPT_automasking_factor_get(
                                 automasking, ss, vertex1, &automask_data);
+
+      automask_data.orig_data.co = cloth_sim->init_pos[v2];
+      automask_data.orig_data.no = cloth_sim->init_no[v2];
       const float mask_v2 = (1.0f - SCULPT_vertex_mask_get(ss, vertex2)) *
                             SCULPT_automasking_factor_get(
                                 automasking, ss, vertex2, &automask_data);
@@ -1078,6 +1087,7 @@ SculptClothSimulation *SCULPT_cloth_brush_simulation_create(Object *ob,
   cloth_sim->last_iteration_pos = MEM_calloc_arrayN(
       totverts, sizeof(float[3]), "cloth sim last iteration pos");
   cloth_sim->init_pos = MEM_calloc_arrayN(totverts, sizeof(float[3]), "cloth sim init pos");
+  cloth_sim->init_no = MEM_calloc_arrayN(totverts, sizeof(float[3]), "cloth sim init normals");
   cloth_sim->length_constraint_tweak = MEM_calloc_arrayN(
       totverts, sizeof(float), "cloth sim length tweak");
 
@@ -1153,6 +1163,7 @@ void SCULPT_cloth_brush_simulation_init(SculptSession *ss, SculptClothSimulation
 
     copy_v3_v3(cloth_sim->last_iteration_pos[i], SCULPT_vertex_co_get(ss, vertex));
     copy_v3_v3(cloth_sim->init_pos[i], SCULPT_vertex_co_get(ss, vertex));
+    SCULPT_vertex_normal_get(ss, vertex, cloth_sim->init_no[i]);
     copy_v3_v3(cloth_sim->prev_pos[i], SCULPT_vertex_co_get(ss, vertex));
     if (has_deformation_pos) {
       copy_v3_v3(cloth_sim->deformation_pos[i], SCULPT_vertex_co_get(ss, vertex));
@@ -1266,6 +1277,7 @@ void SCULPT_cloth_simulation_free(struct SculptClothSimulation *cloth_sim)
   MEM_SAFE_FREE(cloth_sim->deformation_pos);
   MEM_SAFE_FREE(cloth_sim->softbody_pos);
   MEM_SAFE_FREE(cloth_sim->init_pos);
+  MEM_SAFE_FREE(cloth_sim->init_no);
   MEM_SAFE_FREE(cloth_sim->deformation_strength);
   MEM_SAFE_FREE(cloth_sim->node_state);
   BLI_ghash_free(cloth_sim->node_state_index, NULL, NULL);
@@ -1453,6 +1465,8 @@ static void cloth_filter_apply_forces_task_cb(void *__restrict userdata,
 
   PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
     float fade = vd.mask ? *vd.mask : 0.0f;
     fade *= SCULPT_automasking_factor_get(
         ss->filter_cache->automasking, ss, vd.vertex, &automask_data);
@@ -1597,6 +1611,8 @@ static int sculpt_cloth_filter_invoke(bContext *C, wmOperator *op, const wmEvent
   /* Needs mask data to be available as it is used when solving the constraints. */
   BKE_sculpt_update_object_for_edit(depsgraph, ob, true, true, false);
 
+  SCULPT_stroke_id_next(ob);
+
   SCULPT_undo_push_begin(ob, op);
   SCULPT_filter_cache_init(
       C, ob, sd, SCULPT_UNDO_COORDS, event->mval, RNA_float_get(op->ptr, "area_normal_radius"));