Fix T78747: Fix mesh boundary detection and automasking

This issue was produced by a hack in the sculpt mode code from 2.80
 when the sculpt API for connectivity info was not available.
The smooth brush was the only brush that needed connectivity info,
so there were 3 different smooth functions with the connectivity
queries implemented for dyntopo, meshes and grids. The mesh version
of smoothing was checking the number of connected faces to a vertex
to mask the mesh boundaries, which was not covering all cases and
was hardcoded in the smooth function itself.

This patch removes all those legacy functions and unifies all
smooth functions into a single one using the new API and the
automasking system. In order to achieve this, there were needed
some extra changes:

- The smooth brush now does not automasks the boundaries by default,
so its default preset needs to be updated to enable automasking

- The mesh boundary info is extracted once and cached in a
bitmap, similar to the disconnected elements IDs. This makes
boundary detection work as expected in all cases, solving a lot
of known issues with the smooth brush. In multires, this info is
extracted and cached only at the base mesh level, so it is much
more memory efficient than the previous automasking system.

- In order to keep the brushes responsive as they were before,
the automasking system can now skip creating the cache when it
is not needed for the requested options. This means that for
high poly meshes and simple automasking options the brushes
won't lag on start.

Reviewed By: sergey

Maniphest Tasks: T78747

Differential Revision: https://developer.blender.org/D8260

1 files changed, 23 insertions, 197 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_smooth.c b/source/blender/editors/sculpt_paint/sculpt_smooth.c
index 23a20421b74..a616df0a64a 100644
--- a/source/blender/editors/sculpt_paint/sculpt_smooth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_smooth.c
@@ -62,78 +62,29 @@
 #include <math.h>
 #include <stdlib.h>
 
-/* For the smooth brush, uses the neighboring vertices around vert to calculate
- * a smoothed location for vert. Skips corner vertices (used by only one
- * polygon). */
-void SCULPT_neighbor_average(SculptSession *ss, float avg[3], uint vert)
+void SCULPT_neighbor_coords_average_interior(SculptSession *ss, float result[3], int index)
 {
-  const MeshElemMap *vert_map = &ss->pmap[vert];
-  const MVert *mvert = ss->mvert;
-  float(*deform_co)[3] = ss->deform_cos;
-
-  /* Don't modify corner vertices. */
-  if (vert_map->count > 1) {
-    int total = 0;
-
-    zero_v3(avg);
-
-    for (int i = 0; i < vert_map->count; i++) {
-      const MPoly *p = &ss->mpoly[vert_map->indices[i]];
-      uint f_adj_v[2];
-
-      if (poly_get_adj_loops_from_vert(p, ss->mloop, vert, f_adj_v) != -1) {
-        for (int j = 0; j < ARRAY_SIZE(f_adj_v); j += 1) {
-          if (vert_map->count != 2 || ss->pmap[f_adj_v[j]].count <= 2) {
-            add_v3_v3(avg, deform_co ? deform_co[f_adj_v[j]] : mvert[f_adj_v[j]].co);
-
-            total++;
-          }
-        }
-      }
-    }
+  float avg[3] = {0.0f, 0.0f, 0.0f};
+  int total = 0;
 
-    if (total > 0) {
-      mul_v3_fl(avg, 1.0f / total);
-      return;
-    }
+  if (SCULPT_vertex_is_boundary(ss, index)) {
+    copy_v3_v3(result, SCULPT_vertex_co_get(ss, index));
+    return;
   }
 
-  copy_v3_v3(avg, deform_co ? deform_co[vert] : mvert[vert].co);
-}
-
-/* Same logic as neighbor_average(), but for bmesh rather than mesh. */
-void SCULPT_bmesh_neighbor_average(float avg[3], BMVert *v)
-{
-  /* logic for 3 or more is identical. */
-  const int vfcount = BM_vert_face_count_at_most(v, 3);
-
-  /* Don't modify corner vertices. */
-  if (vfcount > 1) {
-    BMIter liter;
-    BMLoop *l;
-    int total = 0;
-
-    zero_v3(avg);
-
-    BM_ITER_ELEM (l, &liter, v, BM_LOOPS_OF_VERT) {
-      const BMVert *adj_v[2] = {l->prev->v, l->next->v};
-
-      for (int i = 0; i < ARRAY_SIZE(adj_v); i++) {
-        const BMVert *v_other = adj_v[i];
-        if (vfcount != 2 || BM_vert_face_count_at_most(v_other, 2) <= 2) {
-          add_v3_v3(avg, v_other->co);
-          total++;
-        }
-      }
-    }
-
-    if (total > 0) {
-      mul_v3_fl(avg, 1.0f / total);
-      return;
-    }
+  SculptVertexNeighborIter ni;
+  SCULPT_VERTEX_NEIGHBORS_ITER_BEGIN (ss, index, ni) {
+    add_v3_v3(avg, SCULPT_vertex_co_get(ss, ni.index));
+    total++;
   }
+  SCULPT_VERTEX_NEIGHBORS_ITER_END(ni);
 
-  copy_v3_v3(avg, v->co);
+  if (total > 0) {
+    mul_v3_v3fl(result, avg, 1.0f / total);
+  }
+  else {
+    copy_v3_v3(result, SCULPT_vertex_co_get(ss, index));
+  }
 }
 
 /* For bmesh: Average surrounding verts based on an orthogonality measure.
@@ -244,125 +195,9 @@ void SCULPT_neighbor_color_average(SculptSession *ss, float result[4], int index
   }
 }
 
-static void do_smooth_brush_mesh_task_cb_ex(void *__restrict userdata,
-                                            const int n,
-                                            const TaskParallelTLS *__restrict tls)
-{
-  SculptThreadedTaskData *data = userdata;
-  SculptSession *ss = data->ob->sculpt;
-  Sculpt *sd = data->sd;
-  const Brush *brush = data->brush;
-  const bool smooth_mask = data->smooth_mask;
-  float bstrength = data->strength;
-
-  PBVHVertexIter vd;
-
-  CLAMP(bstrength, 0.0f, 1.0f);
-
-  SculptBrushTest test;
-  SculptBrushTestFn sculpt_brush_test_sq_fn = SCULPT_brush_test_init_with_falloff_shape(
-      ss, &test, data->brush->falloff_shape);
-
-  const int thread_id = BLI_task_parallel_thread_id(tls);
-
-  BKE_pbvh_vertex_iter_begin(ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE)
-  {
-    if (sculpt_brush_test_sq_fn(&test, vd.co)) {
-      const float fade = bstrength * SCULPT_brush_strength_factor(
-                                         ss,
-                                         brush,
-                                         vd.co,
-                                         sqrtf(test.dist),
-                                         vd.no,
-                                         vd.fno,
-                                         smooth_mask ? 0.0f : (vd.mask ? *vd.mask : 0.0f),
-                                         vd.index,
-                                         thread_id);
-      if (smooth_mask) {
-        float val = SCULPT_neighbor_mask_average(ss, vd.vert_indices[vd.i]) - *vd.mask;
-        val *= fade * bstrength;
-        *vd.mask += val;
-        CLAMP(*vd.mask, 0.0f, 1.0f);
-      }
-      else {
-        float avg[3], val[3];
-
-        SCULPT_neighbor_average(ss, avg, vd.vert_indices[vd.i]);
-        sub_v3_v3v3(val, avg, vd.co);
-
-        madd_v3_v3v3fl(val, vd.co, val, fade);
-
-        SCULPT_clip(sd, ss, vd.co, val);
-      }
-
-      if (vd.mvert) {
-        vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
-      }
-    }
-  }
-  BKE_pbvh_vertex_iter_end;
-}
-
-static void do_smooth_brush_bmesh_task_cb_ex(void *__restrict userdata,
-                                             const int n,
-                                             const TaskParallelTLS *__restrict tls)
-{
-  SculptThreadedTaskData *data = userdata;
-  SculptSession *ss = data->ob->sculpt;
-  Sculpt *sd = data->sd;
-  const Brush *brush = data->brush;
-  const bool smooth_mask = data->smooth_mask;
-  float bstrength = data->strength;
-
-  PBVHVertexIter vd;
-
-  CLAMP(bstrength, 0.0f, 1.0f);
-
-  SculptBrushTest test;
-  SculptBrushTestFn sculpt_brush_test_sq_fn = SCULPT_brush_test_init_with_falloff_shape(
-      ss, &test, data->brush->falloff_shape);
-  const int thread_id = BLI_task_parallel_thread_id(tls);
-
-  BKE_pbvh_vertex_iter_begin(ss->pbvh, data->nodes[n], vd, PBVH_ITER_UNIQUE)
-  {
-    if (sculpt_brush_test_sq_fn(&test, vd.co)) {
-      const float fade = bstrength * SCULPT_brush_strength_factor(ss,
-                                                                  brush,
-                                                                  vd.co,
-                                                                  sqrtf(test.dist),
-                                                                  vd.no,
-                                                                  vd.fno,
-                                                                  smooth_mask ? 0.0f : *vd.mask,
-                                                                  vd.index,
-                                                                  thread_id);
-      if (smooth_mask) {
-        float val = SCULPT_neighbor_mask_average(ss, vd.index) - *vd.mask;
-        val *= fade * bstrength;
-        *vd.mask += val;
-        CLAMP(*vd.mask, 0.0f, 1.0f);
-      }
-      else {
-        float avg[3], val[3];
-
-        SCULPT_bmesh_neighbor_average(avg, vd.bm_vert);
-        sub_v3_v3v3(val, avg, vd.co);
-
-        madd_v3_v3v3fl(val, vd.co, val, fade);
-
-        SCULPT_clip(sd, ss, vd.co, val);
-      }
-
-      if (vd.mvert) {
-        vd.mvert->flag |= ME_VERT_PBVH_UPDATE;
-      }
-    }
-  }
-  BKE_pbvh_vertex_iter_end;
-}
-
-static void do_smooth_brush_multires_task_cb_ex(void *__restrict userdata,
-                                                const int n,
-                                                const TaskParallelTLS *__restrict tls)
+static void do_smooth_brush_task_cb_ex(void *__restrict userdata,
+                                       const int n,
+                                       const TaskParallelTLS *__restrict tls)
 {
   SculptThreadedTaskData *data = userdata;
   SculptSession *ss = data->ob->sculpt;
@@ -438,6 +273,8 @@ void SCULPT_smooth(Sculpt *sd,
     return;
   }
 
+  SCULPT_boundary_info_ensure(ob);
+
   for (iteration = 0; iteration <= count; iteration++) {
     const float strength = (iteration != count) ? 1.0f : last;
 
@@ -452,18 +289,7 @@ void SCULPT_smooth(Sculpt *sd,
 
     TaskParallelSettings settings;
     BKE_pbvh_parallel_range_settings(&settings, true, totnode);
-
-    switch (type) {
-      case PBVH_GRIDS:
-        BLI_task_parallel_range(0, totnode, &data, do_smooth_brush_multires_task_cb_ex, &settings);
-        break;
-      case PBVH_FACES:
-        BLI_task_parallel_range(0, totnode, &data, do_smooth_brush_mesh_task_cb_ex, &settings);
-        break;
-      case PBVH_BMESH:
-        BLI_task_parallel_range(0, totnode, &data, do_smooth_brush_bmesh_task_cb_ex, &settings);
-        break;
-    }
+    BLI_task_parallel_range(0, totnode, &data, do_smooth_brush_task_cb_ex, &settings);
   }
 }