Sculpt: New Cavity Automasking Mode

Add new cavity automasking mode based on local mesh
curvature.  Cavity masking is a great way to quickly add
detail in crevices and the like.  It's meant to be used
with the Paint brush in color attribute mode.  It does
work with other brushes but the results can be unpredictable.

{F13131497}

The old "dirty mask" operator has been replace with a new
"mask from cavity" operator that shares the same code with
cavity automasking.

Differences from the sculpt-dev implementation:
    * It uses the word "cavity."  When I first implemented
this I wasn't aware
      this feature existed in other software (and other
paint modes in Blender),
      and for reasons that escape me today I initially
decided to call it a concave or
      concavity mask.
    * The cavity factor works a bit differently.  It's
      no longer non-linear and functions as a simple
      scale around 0.5f.
    * Supports custom curves.
    * Supports blurring.

Reviewed By: Julian Kaspar, Jeroen Bakker and Campbell Barton
Differential Revision: https://developer.blender.org/D15122
Ref D15122

4 files changed, 19 insertions, 0 deletions

diff --git a/source/blender/makesdna/DNA_brush_defaults.h b/source/blender/makesdna/DNA_brush_defaults.h
index 530c056b584..348e8f4e098 100644
--- a/source/blender/makesdna/DNA_brush_defaults.h
+++ b/source/blender/makesdna/DNA_brush_defaults.h
@@ -91,6 +91,8 @@
     .pose_ik_segments = 1, \
     .hardness = 0.0f, \
     .automasking_boundary_edges_propagation_steps = 1, \
+    .automasking_cavity_blur_steps = 0,\
+    .automasking_cavity_factor = 0.5f,\
  \
     /* A kernel radius of 1 has almost no effect (T63233). */ \
     .blur_kernel_radius = 2, \
diff --git a/source/blender/makesdna/DNA_brush_enums.h b/source/blender/makesdna/DNA_brush_enums.h
index 7a339c3955a..e0567151aa4 100644
--- a/source/blender/makesdna/DNA_brush_enums.h
+++ b/source/blender/makesdna/DNA_brush_enums.h
@@ -330,6 +330,14 @@ typedef enum eAutomasking_flag {
   BRUSH_AUTOMASKING_FACE_SETS = (1 << 1),
   BRUSH_AUTOMASKING_BOUNDARY_EDGES = (1 << 2),
   BRUSH_AUTOMASKING_BOUNDARY_FACE_SETS = (1 << 3),
+  BRUSH_AUTOMASKING_CAVITY_NORMAL = (1 << 4),
+
+  /* Note: normal and inverted are mutually exclusive,
+   * inverted has priority if both bits are set.
+   */
+  BRUSH_AUTOMASKING_CAVITY_INVERTED = (1 << 5),
+  BRUSH_AUTOMASKING_CAVITY_ALL = (1 << 4) | (1 << 5),
+  BRUSH_AUTOMASKING_CAVITY_USE_CURVE = (1 << 6),
 } eAutomasking_flag;
 
 typedef enum ePaintBrush_flag {
diff --git a/source/blender/makesdna/DNA_brush_types.h b/source/blender/makesdna/DNA_brush_types.h
index 761a36e8d1f..c0beb89392b 100644
--- a/source/blender/makesdna/DNA_brush_types.h
+++ b/source/blender/makesdna/DNA_brush_types.h
@@ -388,6 +388,11 @@ typedef struct Brush {
 
   struct BrushGpencilSettings *gpencil_settings;
   struct BrushCurvesSculptSettings *curves_sculpt_settings;
+
+  int automasking_cavity_blur_steps;
+  float automasking_cavity_factor;
+  
+  struct CurveMapping *automasking_cavity_curve;
 } Brush;
 
 /* Struct to hold palette colors for sorting. */
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index c8ff0083b5e..cf163ab7019 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -1021,8 +1021,12 @@ typedef struct Sculpt {
   float constant_detail;
   float detail_percent;
 
+  int automasking_cavity_blur_steps;
+  float automasking_cavity_factor;
   char _pad[4];
 
+  struct CurveMapping *automasking_cavity_curve;
+  struct CurveMapping *automasking_cavity_curve_op; /* For use by operators */
   struct Object *gravity_object;
 } Sculpt;