Fix T96810: Invalid sculpt normals after some operations

Mask and color brushes were using the existing PBVH vertex "update tag"
to mark their modifications. This was mostly unnecessary, and causes
unnecessary calculation of normals. It also caused errors though,
because they didn't tag the corresponding PBVH node for normal
recalculation, causing problems on the borders of nodes, since one
node might accumulate into another's vertex normals, but the other
node wouldn't also accumulate and normalize the normals.

The solution is to only use the update tag for tagging deformed
vertices that need recalculated normals. Everything else is handled at
the PBVH node level (which was already the case, but it wasn't clear).

The update tag was also used for undo to tag the nodes corresponding to
changed vertices. This was wrong though, because normals and visibility
would also be recalculated for just color or mask undo steps. Instead,
just use local arrays to map from vertices to nodes.

Differential Revision: https://developer.blender.org/D15581

1 files changed, 24 insertions, 28 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_brush_types.c b/source/blender/editors/sculpt_paint/sculpt_brush_types.c
index cba97f9b968..c607dd02a77 100644
--- a/source/blender/editors/sculpt_paint/sculpt_brush_types.c
+++ b/source/blender/editors/sculpt_paint/sculpt_brush_types.c
@@ -320,7 +320,7 @@ static void do_draw_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], offset, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -418,7 +418,7 @@ static void do_fill_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], val, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -516,7 +516,7 @@ static void do_scrape_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], val, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -634,7 +634,7 @@ static void do_clay_thumb_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], val, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -789,7 +789,7 @@ static void do_flatten_brush_task_cb_ex(void *__restrict userdata,
       mul_v3_v3fl(proxy[vd.i], val, fade);
 
       if (vd.mvert) {
-        BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+        BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
       }
     }
   }
@@ -946,7 +946,7 @@ static void do_clay_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], val, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1072,7 +1072,7 @@ static void do_clay_strips_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], val, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1272,7 +1272,7 @@ static void do_snake_hook_brush_task_cb_ex(void *__restrict userdata,
     }
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1358,7 +1358,7 @@ static void do_thumb_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], cono, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1436,7 +1436,7 @@ static void do_rotate_brush_task_cb_ex(void *__restrict userdata,
     sub_v3_v3(proxy[vd.i], orig_data.co);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1551,7 +1551,7 @@ static void do_layer_brush_task_cb_ex(void *__restrict userdata,
     SCULPT_clip(sd, ss, vd.co, final_co);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1624,7 +1624,7 @@ static void do_inflate_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3v3(proxy[vd.i], val, ss->cache->scale);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1683,7 +1683,7 @@ static void do_nudge_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], cono, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1777,7 +1777,7 @@ static void do_crease_brush_task_cb_ex(void *__restrict userdata,
     add_v3_v3v3(proxy[vd.i], val1, val2);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -1896,7 +1896,7 @@ static void do_pinch_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], disp_center, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2005,7 +2005,7 @@ static void do_grab_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], grab_delta, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2113,7 +2113,7 @@ static void do_elastic_deform_brush_task_cb_ex(void *__restrict userdata,
     copy_v3_v3(proxy[vd.i], final_disp);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2191,7 +2191,7 @@ static void do_draw_sharp_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], offset, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2304,7 +2304,7 @@ static void do_topology_slide_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], final_disp, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2430,7 +2430,7 @@ static void do_topology_relax_task_cb_ex(void *__restrict userdata,
 
     SCULPT_relax_vertex(ss, &vd, fade * bstrength, false, vd.co);
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2511,7 +2511,7 @@ static void do_displacement_eraser_brush_task_cb_ex(void *__restrict userdata,
     mul_v3_v3fl(proxy[vd.i], disp, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2623,7 +2623,7 @@ static void do_displacement_smear_brush_task_cb_ex(void *__restrict userdata,
     interp_v3_v3v3(vd.co, vd.co, new_co, fade);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2736,7 +2736,7 @@ static void do_topology_rake_bmesh_task_cb_ex(void *__restrict userdata,
     SCULPT_clip(sd, ss, vd.co, val);
 
     if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
+      BKE_pbvh_vert_tag_update_normal(ss->pbvh, vd.index);
     }
   }
   BKE_pbvh_vertex_iter_end;
@@ -2808,12 +2808,8 @@ static void do_mask_brush_draw_task_cb_ex(void *__restrict userdata,
       (*vd.mask) += fade * bstrength * (*vd.mask);
     }
     *vd.mask = clamp_f(*vd.mask, 0.0f, 1.0f);
-
-    if (vd.mvert) {
-      BKE_pbvh_vert_mark_update(ss->pbvh, vd.index);
-    }
-    BKE_pbvh_vertex_iter_end;
   }
+  BKE_pbvh_vertex_iter_end;
 }
 
 void SCULPT_do_mask_brush_draw(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode)