Fix T78747: Fix mesh boundary detection and automasking

This issue was produced by a hack in the sculpt mode code from 2.80
 when the sculpt API for connectivity info was not available.
The smooth brush was the only brush that needed connectivity info,
so there were 3 different smooth functions with the connectivity
queries implemented for dyntopo, meshes and grids. The mesh version
of smoothing was checking the number of connected faces to a vertex
to mask the mesh boundaries, which was not covering all cases and
was hardcoded in the smooth function itself.

This patch removes all those legacy functions and unifies all
smooth functions into a single one using the new API and the
automasking system. In order to achieve this, there were needed
some extra changes:

- The smooth brush now does not automasks the boundaries by default,
so its default preset needs to be updated to enable automasking

- The mesh boundary info is extracted once and cached in a
bitmap, similar to the disconnected elements IDs. This makes
boundary detection work as expected in all cases, solving a lot
of known issues with the smooth brush. In multires, this info is
extracted and cached only at the base mesh level, so it is much
more memory efficient than the previous automasking system.

- In order to keep the brushes responsive as they were before,
the automasking system can now skip creating the cache when it
is not needed for the requested options. This means that for
high poly meshes and simple automasking options the brushes
won't lag on start.

Reviewed By: sergey

Maniphest Tasks: T78747

Differential Revision: https://developer.blender.org/D8260

1 files changed, 20 insertions, 8 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index e943bd280a3..17c7e9e86c0 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -148,8 +148,6 @@ int SCULPT_active_vertex_get(SculptSession *ss);
 const float *SCULPT_active_vertex_co_get(SculptSession *ss);
 void SCULPT_active_vertex_normal_get(SculptSession *ss, float normal[3]);
 
-bool SCULPT_vertex_is_boundary(SculptSession *ss, const int index);
-
 /* Fake Neighbors */
 
 #define FAKE_NEIGHBOR_NONE -1
@@ -159,6 +157,11 @@ void SCULPT_fake_neighbors_enable(Object *ob);
 void SCULPT_fake_neighbors_disable(Object *ob);
 void SCULPT_fake_neighbors_free(struct Object *ob);
 
+/* Vertex Info. */
+void SCULPT_boundary_info_ensure(Object *object);
+/* Boundary Info needs to be initialized in order to use this function. */
+bool SCULPT_vertex_is_boundary(const SculptSession *ss, const int index);
+
 /* Sculpt Visibility API */
 
 void SCULPT_vertex_visible_set(SculptSession *ss, int index, bool visible);
@@ -179,7 +182,7 @@ bool SCULPT_vertex_has_unique_face_set(SculptSession *ss, int index);
 int SCULPT_face_set_next_available_get(SculptSession *ss);
 
 void SCULPT_face_set_visibility_set(SculptSession *ss, int face_set, bool visible);
-bool SCULPT_vertex_all_face_sets_visible_get(SculptSession *ss, int index);
+bool SCULPT_vertex_all_face_sets_visible_get(const SculptSession *ss, int index);
 bool SCULPT_vertex_any_face_set_visible_get(SculptSession *ss, int index);
 
 void SCULPT_face_sets_visibility_invert(SculptSession *ss);
@@ -404,16 +407,15 @@ void SCULPT_do_paint_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode
 void SCULPT_do_smear_brush(Sculpt *sd, Object *ob, PBVHNode **nodes, int totnode);
 
 /* Smooth Brush. */
-
-void SCULPT_neighbor_average(SculptSession *ss, float avg[3], uint vert);
-void SCULPT_bmesh_neighbor_average(float avg[3], struct BMVert *v);
-
 void SCULPT_bmesh_four_neighbor_average(float avg[3], float direction[3], struct BMVert *v);
 
 void SCULPT_neighbor_coords_average(SculptSession *ss, float result[3], int index);
 float SCULPT_neighbor_mask_average(SculptSession *ss, int index);
 void SCULPT_neighbor_color_average(SculptSession *ss, float result[4], int index);
 
+/* Mask the mesh boundaries smoothing only the mesh surface without using automasking. */
+void SCULPT_neighbor_coords_average_interior(SculptSession *ss, float result[3], int index);
+
 void SCULPT_smooth(Sculpt *sd,
                    Object *ob,
                    PBVHNode **nodes,
@@ -735,6 +737,12 @@ bool SCULPT_pbvh_calc_area_normal(const struct Brush *brush,
 
 #define SCULPT_CLAY_STABILIZER_LEN 10
 
+typedef struct AutomaskingSettings {
+  /* Flags from eAutomasking_flag. */
+  int flags;
+  int initial_face_set;
+} AutomaskingSettings;
+
 typedef struct StrokeCache {
   /* Invariants */
   float initial_radius;
@@ -857,7 +865,11 @@ typedef struct StrokeCache {
   float true_gravity_direction[3];
   float gravity_direction[3];
 
-  float *automask;
+  /* Automasking. */
+  AutomaskingSettings automask_settings;
+  /* Precomputed automask factor indexed by vertex, owned by the automasking system and initialized
+   * in SCULPT_automasking_init when needed. */
+  float *automask_factor;
 
   float stroke_local_mat[4][4];
   float multiplane_scrape_angle;