Sculpt: Add global automasking settings support in filters

When using the sculpt filters, global automasking settings that affect
all brushes were ignored because the automasking system was not
implemented for filters, making filters and brushes react differently
to the global sculpt settings which creates confusion.

This makes all filter tools (mesh, cloth, color) use the same general
automasking settings and features as the brush tools. Filters will now
use the settings in the options panel to limit their effect.

This also removes the "use Face Sets" option from the Mesh filter code,
as it was duplicated from the automasking code just to have that
funcitonality. This is now handled by the regular automasking system.

The "Use Face Sets" option is still available in the cloth filter as that
option limits the action of the forces, not the displacement.

After this, it is possible to initialize the automasking system
independently from the StrokeCache and Brush settings, so it can also be
added to more tools and features in the future.

Fixes T81619

Reviewed By: dbystedt, sergey

Maniphest Tasks: T81619

Differential Revision: https://developer.blender.org/D9171

1 files changed, 68 insertions, 50 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt_automasking.c b/source/blender/editors/sculpt_paint/sculpt_automasking.c
index 241f1167316..f2fc1bcb3c9 100644
--- a/source/blender/editors/sculpt_paint/sculpt_automasking.c
+++ b/source/blender/editors/sculpt_paint/sculpt_automasking.c
@@ -62,16 +62,30 @@
 #include <math.h>
 #include <stdlib.h>
 
+AutomaskingCache *SCULPT_automasking_active_cache_get(SculptSession *ss)
+{
+  if (ss->cache) {
+    return ss->cache->automasking;
+  }
+  else if (ss->filter_cache) {
+    return ss->filter_cache->automasking;
+  }
+  return NULL;
+}
+
 bool SCULPT_is_automasking_mode_enabled(const Sculpt *sd,
                                         const Brush *br,
                                         const eAutomasking_flag mode)
 {
-  return br->automasking_flags & mode || sd->automasking_flags & mode;
+  if (br) {
+    return br->automasking_flags & mode || sd->automasking_flags & mode;
+  }
+  return sd->automasking_flags & mode;
 }
 
 bool SCULPT_is_automasking_enabled(const Sculpt *sd, const SculptSession *ss, const Brush *br)
 {
-  if (SCULPT_stroke_is_dynamic_topology(ss, br)) {
+  if (br && SCULPT_stroke_is_dynamic_topology(ss, br)) {
     return false;
   }
   if (SCULPT_is_automasking_mode_enabled(sd, br, BRUSH_AUTOMASKING_TOPOLOGY)) {
@@ -91,10 +105,13 @@ bool SCULPT_is_automasking_enabled(const Sculpt *sd, const SculptSession *ss, co
 
 static int sculpt_automasking_mode_effective_bits(const Sculpt *sculpt, const Brush *brush)
 {
-  return sculpt->automasking_flags | brush->automasking_flags;
+  if (brush) {
+    return sculpt->automasking_flags | brush->automasking_flags;
+  }
+  return sculpt->automasking_flags;
 }
 
-static bool SCULPT_automasking_needs_cache(const Sculpt *sd, const Brush *brush)
+static bool SCULPT_automasking_needs_factors_cache(const Sculpt *sd, const Brush *brush)
 {
 
   const int automasking_flags = sculpt_automasking_mode_effective_bits(sd, brush);
@@ -102,39 +119,39 @@ static bool SCULPT_automasking_needs_cache(const Sculpt *sd, const Brush *brush)
     return true;
   }
   if (automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
-    return brush->automasking_boundary_edges_propagation_steps != 1;
+    return brush && brush->automasking_boundary_edges_propagation_steps != 1;
   }
   if (automasking_flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
-    return brush->automasking_boundary_edges_propagation_steps != 1;
+    return brush && brush->automasking_boundary_edges_propagation_steps != 1;
   }
   return false;
 }
 
-float SCULPT_automasking_factor_get(SculptSession *ss, int vert)
+float SCULPT_automasking_factor_get(AutomaskingCache *automasking, SculptSession *ss, int vert)
 {
-  if (!ss->cache) {
+  if (!automasking) {
     return 1.0f;
   }
   /* If the cache is initialized with valid info, use the cache. This is used when the
    * automasking information can't be computed in real time per vertex and needs to be
    * initialized for the whole mesh when the stroke starts. */
-  if (ss->cache->automask_factor) {
-    return ss->cache->automask_factor[vert];
+  if (automasking->factor) {
+    return automasking->factor[vert];
   }
 
-  if (ss->cache->automask_settings.flags & BRUSH_AUTOMASKING_FACE_SETS) {
-    if (!SCULPT_vertex_has_face_set(ss, vert, ss->cache->automask_settings.initial_face_set)) {
+  if (automasking->settings.flags & BRUSH_AUTOMASKING_FACE_SETS) {
+    if (!SCULPT_vertex_has_face_set(ss, vert, automasking->settings.initial_face_set)) {
       return 0.0f;
     }
   }
 
-  if (ss->cache->automask_settings.flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
+  if (automasking->settings.flags & BRUSH_AUTOMASKING_BOUNDARY_EDGES) {
     if (SCULPT_vertex_is_boundary(ss, vert)) {
       return 0.0f;
     }
   }
 
-  if (ss->cache->automask_settings.flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
+  if (automasking->settings.flags & BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS) {
     if (!SCULPT_vertex_has_unique_face_set(ss, vert)) {
       return 0.0f;
     }
@@ -143,12 +160,14 @@ float SCULPT_automasking_factor_get(SculptSession *ss, int vert)
   return 1.0f;
 }
 
-void SCULPT_automasking_end(Object *ob)
+void SCULPT_automasking_cache_free(AutomaskingCache *automasking)
 {
-  SculptSession *ss = ob->sculpt;
-  if (ss->cache && ss->cache->automask_factor) {
-    MEM_freeN(ss->cache->automask_factor);
+  if (!automasking) {
+    return;
   }
+
+  MEM_SAFE_FREE(automasking->factor);
+  MEM_SAFE_FREE(automasking);
 }
 
 static bool sculpt_automasking_is_constrained_by_radius(Brush *br)
@@ -189,10 +208,6 @@ static float *SCULPT_topology_automasking_init(Sculpt *sd, Object *ob, float *au
   SculptSession *ss = ob->sculpt;
   Brush *brush = BKE_paint_brush(&sd->paint);
 
-  if (!SCULPT_is_automasking_enabled(sd, ss, brush)) {
-    return NULL;
-  }
-
   if (BKE_pbvh_type(ss->pbvh) == PBVH_FACES && !ss->pmap) {
     BLI_assert(!"Topology masking: pmap missing");
     return NULL;
@@ -200,19 +215,20 @@ static float *SCULPT_topology_automasking_init(Sculpt *sd, Object *ob, float *au
 
   const int totvert = SCULPT_vertex_count_get(ss);
   for (int i = 0; i < totvert; i++) {
-    ss->cache->automask_factor[i] = 0.0f;
+    automask_factor[i] = 0.0f;
   }
 
   /* Flood fill automask to connected vertices. Limited to vertices inside
    * the brush radius if the tool requires it. */
   SculptFloodFill flood;
   SCULPT_floodfill_init(ss, &flood);
-  SCULPT_floodfill_add_active(sd, ob, ss, &flood, ss->cache->radius);
+  const float radius = ss->cache ? ss->cache->radius : FLT_MAX;
+  SCULPT_floodfill_add_active(sd, ob, ss, &flood, radius);
 
   AutomaskFloodFillData fdata = {
       .automask_factor = automask_factor,
-      .radius = ss->cache->radius,
-      .use_radius = sculpt_automasking_is_constrained_by_radius(brush),
+      .radius = radius,
+      .use_radius = ss->cache && sculpt_automasking_is_constrained_by_radius(brush),
       .symm = SCULPT_mesh_symmetry_xyz_get(ob),
   };
   copy_v3_v3(fdata.location, SCULPT_active_vertex_co_get(ss));
@@ -306,58 +322,60 @@ float *SCULPT_boundary_automasking_init(Object *ob,
   return automask_factor;
 }
 
-static void SCULPT_stroke_automasking_settings_update(SculptSession *ss, Sculpt *sd, Brush *brush)
+static void SCULPT_automasking_cache_settings_update(AutomaskingCache *automasking,
+                                                     SculptSession *ss,
+                                                     Sculpt *sd,
+                                                     Brush *brush)
 {
-  BLI_assert(ss->cache);
-
-  ss->cache->automask_settings.flags = sculpt_automasking_mode_effective_bits(sd, brush);
-  ss->cache->automask_settings.initial_face_set = SCULPT_active_face_set_get(ss);
+  automasking->settings.flags = sculpt_automasking_mode_effective_bits(sd, brush);
+  automasking->settings.initial_face_set = SCULPT_active_face_set_get(ss);
 }
 
-void SCULPT_automasking_init(Sculpt *sd, Object *ob)
+AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object *ob)
 {
   SculptSession *ss = ob->sculpt;
-  Brush *brush = BKE_paint_brush(&sd->paint);
   const int totvert = SCULPT_vertex_count_get(ss);
 
   if (!SCULPT_is_automasking_enabled(sd, ss, brush)) {
-    return;
+    return NULL;
   }
 
-  SCULPT_stroke_automasking_settings_update(ss, sd, brush);
+  AutomaskingCache *automasking = MEM_callocN(sizeof(AutomaskingCache), "automasking cache");
+  SCULPT_automasking_cache_settings_update(automasking, ss, sd, brush);
   SCULPT_boundary_info_ensure(ob);
 
-  if (!SCULPT_automasking_needs_cache(sd, brush)) {
-    return;
+  if (!SCULPT_automasking_needs_factors_cache(sd, brush)) {
+    return automasking;
   }
 
-  ss->cache->automask_factor = MEM_malloc_arrayN(totvert, sizeof(float), "automask_factor");
-
+  automasking->factor = MEM_malloc_arrayN(totvert, sizeof(float), "automask_factor");
   for (int i = 0; i < totvert; i++) {
-    ss->cache->automask_factor[i] = 1.0f;
+    automasking->factor[i] = 1.0f;
   }
 
+  const int boundary_propagation_steps = brush ?
+                                             brush->automasking_boundary_edges_propagation_steps :
+                                             1;
+
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_TOPOLOGY)) {
     SCULPT_vertex_random_access_ensure(ss);
-    SCULPT_topology_automasking_init(sd, ob, ss->cache->automask_factor);
+    SCULPT_topology_automasking_init(sd, ob, automasking->factor);
   }
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_FACE_SETS)) {
     SCULPT_vertex_random_access_ensure(ss);
-    sculpt_face_sets_automasking_init(sd, ob, ss->cache->automask_factor);
+    sculpt_face_sets_automasking_init(sd, ob, automasking->factor);
   }
 
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_BOUNDARY_EDGES)) {
     SCULPT_vertex_random_access_ensure(ss);
-    SCULPT_boundary_automasking_init(ob,
-                                     AUTOMASK_INIT_BOUNDARY_EDGES,
-                                     brush->automasking_boundary_edges_propagation_steps,
-                                     ss->cache->automask_factor);
+    SCULPT_boundary_automasking_init(
+        ob, AUTOMASK_INIT_BOUNDARY_EDGES, boundary_propagation_steps, automasking->factor);
   }
   if (SCULPT_is_automasking_mode_enabled(sd, brush, BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS)) {
     SCULPT_vertex_random_access_ensure(ss);
-    SCULPT_boundary_automasking_init(ob,
-                                     AUTOMASK_INIT_BOUNDARY_FACE_SETS,
-                                     brush->automasking_boundary_edges_propagation_steps,
-                                     ss->cache->automask_factor);
+    SCULPT_boundary_automasking_init(
+        ob, AUTOMASK_INIT_BOUNDARY_FACE_SETS, boundary_propagation_steps, automasking->factor);
   }
+
+  return automasking;
 }