Fix T80008: Smooth brush not deforming mesh boundaries correctly

In 2.83 and previous versions there was a bug that was causing boundary
vertices to be detected incorrectly that was preventing the smooth brush
to work on boundaries if there was a pole on them.
In 2.90 the boundary vertex detection was fixed, but it was still using a
simplified version of the algorithm without any boundary smoothing. This
patch implements a similar smoothing algorithm to what I think it was
the intention of 2.83 and previous versions, but working correctly.

Reviewed By: sergey

Maniphest Tasks: T80008

Differential Revision: https://developer.blender.org/D8680

2 files changed, 26 insertions, 12 deletions

diff --git a/source/blender/blenkernel/intern/brush.c b/source/blender/blenkernel/intern/brush.c
index 8cd30c2241f..3c83ee55213 100644
--- a/source/blender/blenkernel/intern/brush.c
+++ b/source/blender/blenkernel/intern/brush.c
@@ -1539,7 +1539,6 @@ void BKE_brush_sculpt_reset(Brush *br)
       break;
     case SCULPT_TOOL_SMOOTH:
       br->flag &= ~BRUSH_SPACE_ATTEN;
-      br->automasking_flags |= BRUSH_AUTOMASKING_BOUNDARY_EDGES;
       br->spacing = 5;
       br->alpha = 0.7f;
       br->surface_smooth_shape_preservation = 0.5f;
diff --git a/source/blender/editors/sculpt_paint/sculpt_smooth.c b/source/blender/editors/sculpt_paint/sculpt_smooth.c
index 63fe8643628..87ee7480c92 100644
--- a/source/blender/editors/sculpt_paint/sculpt_smooth.c
+++ b/source/blender/editors/sculpt_paint/sculpt_smooth.c
@@ -66,25 +66,40 @@ void SCULPT_neighbor_coords_average_interior(SculptSession *ss, float result[3],
 {
   float avg[3] = {0.0f, 0.0f, 0.0f};
   int total = 0;
-
-  if (SCULPT_vertex_is_boundary(ss, index)) {
-    copy_v3_v3(result, SCULPT_vertex_co_get(ss, index));
-    return;
-  }
+  int neighbor_count = 0;
+  const bool is_boundary = SCULPT_vertex_is_boundary(ss, index);
 
   SculptVertexNeighborIter ni;
   SCULPT_VERTEX_NEIGHBORS_ITER_BEGIN (ss, index, ni) {
-    add_v3_v3(avg, SCULPT_vertex_co_get(ss, ni.index));
-    total++;
+    neighbor_count++;
+    if (is_boundary) {
+      /* Boundary vertices use only other boundary vertices. */
+      if (SCULPT_vertex_is_boundary(ss, ni.index)) {
+        add_v3_v3(avg, SCULPT_vertex_co_get(ss, ni.index));
+        total++;
+      }
+    }
+    else {
+      /* Interior vertices use all neighbors. */
+      add_v3_v3(avg, SCULPT_vertex_co_get(ss, ni.index));
+      total++;
+    }
   }
   SCULPT_VERTEX_NEIGHBORS_ITER_END(ni);
 
-  if (total > 0) {
-    mul_v3_v3fl(result, avg, 1.0f / total);
+  /* Do not modify corner vertices. */
+  if (neighbor_count <= 2) {
+    copy_v3_v3(result, SCULPT_vertex_co_get(ss, index));
+    return;
   }
-  else {
+
+  /* Avoid division by 0 when there are no neighbors. */
+  if (total == 0) {
     copy_v3_v3(result, SCULPT_vertex_co_get(ss, index));
+    return;
   }
+
+  mul_v3_v3fl(result, avg, 1.0f / total);
 }
 
 /* For bmesh: Average surrounding verts based on an orthogonality measure.
@@ -316,7 +331,7 @@ static void do_smooth_brush_task_cb_ex(void *__restrict userdata,
       }
       else {
         float avg[3], val[3];
-        SCULPT_neighbor_coords_average(ss, avg, vd.index);
+        SCULPT_neighbor_coords_average_interior(ss, avg, vd.index);
         sub_v3_v3v3(val, avg, vd.co);
         madd_v3_v3v3fl(val, vd.co, val, fade);
         SCULPT_clip(sd, ss, vd.co, val);