Fix Sculpt Relax operation when deforming mesh boundaries

Previously, mesh boundaries were relaxed as any other vertex, which was
causing artifacts and unwanted deformation. In order to prevent this,
the mesh filter was using the automasking system to lock the boundary
vertices, which was hacked into the tool. For the brush, the only
solution was to enable boundary automasking to lock those vertices
in plance.

Now the relax vertex function slides the boundary vertices along the
mesh boundary edges, relaxing all the topology correctly while
preserving the shape of the mesh. The automasking hack in the relax
mesh filter was also removed as now vertices slide correctly along
the boundary.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D8350

1 files changed, 5 insertions, 12 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
index 9c9726ff3db..e9a98a17f8a 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_mesh.c
@@ -310,8 +310,7 @@ static void mesh_filter_task_cb(void *__restrict userdata,
         break;
       }
       case MESH_FILTER_RELAX: {
-        SCULPT_relax_vertex(
-            ss, &vd, clamp_f(fade * ss->filter_cache->automask[vd.index], 0.0f, 1.0f), false, val);
+        SCULPT_relax_vertex(ss, &vd, clamp_f(fade, 0.0f, 1.0f), false, val);
         sub_v3_v3v3(disp, val, vd.co);
         break;
       }
@@ -543,6 +542,10 @@ static int sculpt_mesh_filter_invoke(bContext *C, wmOperator *op, const wmEvent
 
   SCULPT_undo_push_begin("Mesh filter");
 
+  if (ELEM(filter_type, MESH_FILTER_RELAX, MESH_FILTER_RELAX_FACE_SETS)) {
+    SCULPT_boundary_info_ensure(ob);
+  }
+
   SCULPT_filter_cache_init(ob, sd, SCULPT_UNDO_COORDS);
 
   if (use_face_sets) {
@@ -572,16 +575,6 @@ static int sculpt_mesh_filter_invoke(bContext *C, wmOperator *op, const wmEvent
   ss->filter_cache->enabled_axis[1] = deform_axis & MESH_FILTER_DEFORM_Y;
   ss->filter_cache->enabled_axis[2] = deform_axis & MESH_FILTER_DEFORM_Z;
 
-  if (RNA_enum_get(op->ptr, "type") == MESH_FILTER_RELAX) {
-    ss->filter_cache->automask = MEM_mallocN(totvert * sizeof(float),
-                                             "Relax filter edge automask");
-    for (int i = 0; i < totvert; i++) {
-      ss->filter_cache->automask[i] = 1.0f;
-    }
-    SCULPT_boundary_automasking_init(
-        ob, AUTOMASK_INIT_BOUNDARY_EDGES, 1, ss->filter_cache->automask);
-  }
-
   WM_event_add_modal_handler(C, op);
   return OPERATOR_RUNNING_MODAL;
 }