Fix T79054: Crash with Cloth Brush and anchored stroke

As the comment says, anchored stroke can't rely on the first stroke
iteration for creating the simulation data. Probably lost in a cleanup.

I also made that anchored stroke doesn't restore the mesh state in the
cloth brush, so it can create the simulation effect.

Reviewed By: sergey

Maniphest Tasks: T79054

Differential Revision: https://developer.blender.org/D8348

2 files changed, 12 insertions, 6 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 590d04bed3d..92beb396629 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -5516,7 +5516,7 @@ static void do_brush_action(Sculpt *sd, Object *ob, Brush *brush, UnifiedPaintSe
     SculptSearchSphereData data = {
         .ss = ss,
         .sd = sd,
-        .radius_squared = square_f(ss->cache->radius * (1.0 + brush->cloth_sim_limit)),
+        .radius_squared = square_f(ss->cache->initial_radius * (1.0 + brush->cloth_sim_limit)),
         .original = false,
         .ignore_fully_ineffective = false,
         .center = ss->cache->initial_location,
@@ -7208,11 +7208,16 @@ static void sculpt_restore_mesh(Sculpt *sd, Object *ob)
   SculptSession *ss = ob->sculpt;
   Brush *brush = BKE_paint_brush(&sd->paint);
 
+  /* For the cloth brush it makes more sense to not restore the mesh state to keep running the
+   * simulation from the previous state. */
+  if (brush->sculpt_tool == SCULPT_TOOL_CLOTH) {
+    return;
+  }
+
   /* Restore the mesh before continuing with anchored stroke. */
   if ((brush->flag & BRUSH_ANCHORED) ||
       ((brush->sculpt_tool == SCULPT_TOOL_GRAB ||
-        brush->sculpt_tool == SCULPT_TOOL_ELASTIC_DEFORM ||
-        brush->sculpt_tool == SCULPT_TOOL_CLOTH) &&
+        brush->sculpt_tool == SCULPT_TOOL_ELASTIC_DEFORM) &&
        BKE_brush_use_size_pressure(brush)) ||
       (brush->flag & BRUSH_DRAG_DOT)) {
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_cloth.c b/source/blender/editors/sculpt_paint/sculpt_cloth.c
index 6a2137b6626..115d6f665a3 100644
--- a/source/blender/editors/sculpt_paint/sculpt_cloth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_cloth.c
@@ -639,12 +639,13 @@ void SCULPT_do_cloth_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode
 
   /* In the first brush step of each symmetry pass, build the constraints for the vertices in all
    * nodes inside the simulation's limits. */
-  /* Brush stroke types that restore the mesh on each brush step also need the cloth sim data to be
-   * created on each step. */
+  /* Brushes that use anchored strokes and restore the mesh can't rely on symmetry passes and steps
+   * count as it is always the first step, so the simulation needs to be created when it does not
+   * exist for this stroke. */
   if (SCULPT_stroke_is_first_brush_step_of_symmetry_pass(ss->cache) || !ss->cache->cloth_sim) {
 
     /* The simulation structure only needs to be created on the first symmetry pass. */
-    if (SCULPT_stroke_is_first_brush_step(ss->cache)) {
+    if (SCULPT_stroke_is_first_brush_step(ss->cache) || !ss->cache->cloth_sim) {
       ss->cache->cloth_sim = cloth_brush_simulation_create(
           ss, brush->cloth_mass, brush->cloth_damping);
       for (int i = 0; i < totverts; i++) {