temp-texpaint-automasking: Support automasking in new paint

* New texture paint now supports automasking.
* Automasking factor cache can now be built incrementally
  by PBVH node.

7 files changed, 204 insertions, 17 deletions

diff --git a/source/blender/blenkernel/BKE_pbvh.h b/source/blender/blenkernel/BKE_pbvh.h
index f517ff3a949..380820030b1 100644
--- a/source/blender/blenkernel/BKE_pbvh.h
+++ b/source/blender/blenkernel/BKE_pbvh.h
@@ -79,6 +79,7 @@ typedef enum {
   PBVH_UpdateTopology = 1 << 13,
   PBVH_UpdateColor = 1 << 14,
   PBVH_RebuildPixels = 1 << 15,
+  PBVH_RebuildAutomasking = 1 << 16,
 
 } PBVHNodeFlags;
 
@@ -413,7 +414,7 @@ typedef struct PBVHVertexIter {
   /* mesh */
   struct MVert *mverts;
   float (*vert_normals)[3];
-  int totvert;
+  int totvert, unique_verts;
   const int *vert_indices;
   float *vmask;
 
@@ -587,6 +588,23 @@ void BKE_pbvh_vertex_color_get(const PBVH *pbvh, int vertex, float r_color[4]);
 void BKE_pbvh_ensure_node_loops(PBVH *pbvh);
 bool BKE_pbvh_draw_cache_invalid(const PBVH *pbvh);
 
+void BKE_pbvh_node_automasking_mark(PBVH *pbvh, PBVHNode *node);
+void BKE_pbvh_node_automasking_unmark(PBVH *pbvh, PBVHNode *node);
+bool BKE_pbvh_node_needs_automasking(PBVH *pbvh, PBVHNode *node);
+void BKE_pbvh_node_automasking_mark_all(PBVH *pbvh);
+
+/* XXX Temporary attribute for patch development; remove for final patch! */
+#ifdef __clang__
+#  define ATTR_NO_OPT __attribute__((optnone))
+#elif defined(_MSC_VER)
+#  define ATTR_NO_OPT __pragma(optimize("", off))
+#elif defined(__GNUC__)
+#  define ATTR_NO_OPT __attribute__((optimize("O0")))
+#else
+#  define ATTR_NO_OPT
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/blenkernel/intern/pbvh.c b/source/blender/blenkernel/intern/pbvh.c
index 3b20bdc826c..88cfa6c5db2 100644
--- a/source/blender/blenkernel/intern/pbvh.c
+++ b/source/blender/blenkernel/intern/pbvh.c
@@ -3126,6 +3126,9 @@ void pbvh_vertex_iter_init(PBVH *pbvh, PBVHNode *node, PBVHVertexIter *vi, int m
   else {
     vi->totvert = uniq_verts;
   }
+
+  vi->unique_verts = uniq_verts;
+
   vi->vert_indices = vert_indices;
   vi->mverts = verts;
 
@@ -3320,3 +3323,27 @@ void BKE_pbvh_ensure_node_loops(PBVH *pbvh)
 
   MEM_SAFE_FREE(visit);
 }
+
+void BKE_pbvh_node_automasking_mark(PBVH *pbvh, PBVHNode *node)
+{
+  node->flag |= PBVH_RebuildAutomasking;
+}
+
+void BKE_pbvh_node_automasking_unmark(PBVH *pbvh, PBVHNode *node)
+{
+  node->flag &= ~PBVH_RebuildAutomasking;
+}
+
+bool BKE_pbvh_node_needs_automasking(PBVH *pbvh, PBVHNode *node)
+{
+  return node->flag & PBVH_RebuildAutomasking;
+}
+
+void BKE_pbvh_node_automasking_mark_all(PBVH *pbvh)
+{
+  for (int i = 0; i < pbvh->totnode; i++) {
+    if (pbvh->nodes[i].flag & PBVH_Leaf) {
+      pbvh->nodes[i].flag |= PBVH_RebuildAutomasking;
+    }
+  }
+}
diff --git a/source/blender/blenkernel/intern/pbvh_intern.h b/source/blender/blenkernel/intern/pbvh_intern.h
index a4ac2744a73..9f6f02334a2 100644
--- a/source/blender/blenkernel/intern/pbvh_intern.h
+++ b/source/blender/blenkernel/intern/pbvh_intern.h
@@ -96,7 +96,7 @@ struct PBVHNode {
 
   /* Indicates whether this node is a leaf or not; also used for
    * marking various updates that need to be applied. */
-  PBVHNodeFlags flag : 16;
+  PBVHNodeFlags flag : 32;
 
   /* Used for raycasting: how close bb is to the ray point. */
   float tmin;
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 5ea42a5efa5..5e6259e3e07 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -1288,7 +1288,10 @@ void SCULPT_orig_vert_data_unode_init(SculptOrigVertData *data, Object *ob, Scul
   }
 }
 
-void SCULPT_orig_vert_data_init(SculptOrigVertData *data, Object *ob, PBVHNode *node, SculptUndoType type)
+void SCULPT_orig_vert_data_init(SculptOrigVertData *data,
+                                Object *ob,
+                                PBVHNode *node,
+                                SculptUndoType type)
 {
   SculptUndoNode *unode;
   unode = SCULPT_undo_push_node(ob, node, type);
@@ -2447,7 +2450,9 @@ float SCULPT_brush_strength_factor(SculptSession *ss,
   avg *= 1.0f - mask;
 
   /* Auto-masking. */
-  avg *= SCULPT_automasking_factor_get(cache->automasking, ss, vertex_index);
+  if (vertex_index != -1) {
+    avg *= SCULPT_automasking_factor_get(cache->automasking, ss, vertex_index);
+  }
 
   return avg;
 }
@@ -4173,6 +4178,7 @@ static void sculpt_update_cache_invariants(
 {
   StrokeCache *cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
   UnifiedPaintSettings *ups = &CTX_data_tool_settings(C)->unified_paint_settings;
+  ToolSettings *tool_settings = CTX_data_tool_settings(C);
   Brush *brush = BKE_paint_brush(&sd->paint);
   ViewContext *vc = paint_stroke_view_context(op->customdata);
   Object *ob = CTX_data_active_object(C);
@@ -4183,6 +4189,8 @@ static void sculpt_update_cache_invariants(
 
   ss->cache = cache;
 
+  cache->use_pixels = sculpt_needs_pbvh_pixels(&tool_settings->paint_mode, brush, ob);
+
   /* Set scaling adjustment. */
   max_scale = 0.0f;
   for (int i = 0; i < 3; i++) {
@@ -5334,6 +5342,11 @@ static bool sculpt_stroke_test_start(bContext *C, struct wmOperator *op, const f
       SCULPT_undo_push_begin(ob, sculpt_tool_name(sd));
     }
 
+    /* Mark all pbvh nodes for update. */
+    if (ss->pbvh) {
+      BKE_pbvh_node_automasking_mark_all(ss->pbvh);
+    }
+
     return true;
   }
   return false;
diff --git a/source/blender/editors/sculpt_paint/sculpt_automasking.cc b/source/blender/editors/sculpt_paint/sculpt_automasking.cc
index bb101717c9b..4d40f473fd3 100644
--- a/source/blender/editors/sculpt_paint/sculpt_automasking.cc
+++ b/source/blender/editors/sculpt_paint/sculpt_automasking.cc
@@ -11,7 +11,10 @@
 #include "BLI_hash.h"
 #include "BLI_index_range.hh"
 #include "BLI_math.h"
+#include "BLI_set.hh"
 #include "BLI_task.h"
+#include "BLI_task.hh"
+#include "BLI_vector.hh"
 
 #include "DNA_brush_types.h"
 #include "DNA_mesh_types.h"
@@ -48,6 +51,8 @@
 #include <cstdlib>
 
 using blender::IndexRange;
+using blender::Set;
+using blender::Vector;
 
 AutomaskingCache *SCULPT_automasking_active_cache_get(SculptSession *ss)
 {
@@ -114,18 +119,8 @@ static bool SCULPT_automasking_needs_factors_cache(const Sculpt *sd, const Brush
   return false;
 }
 
-float SCULPT_automasking_factor_get(AutomaskingCache *automasking, SculptSession *ss, int vert)
+float sculpt_automasking_factor_calc(AutomaskingCache *automasking, SculptSession *ss, int vert)
 {
-  if (!automasking) {
-    return 1.0f;
-  }
-  /* If the cache is initialized with valid info, use the cache. This is used when the
-   * automasking information can't be computed in real time per vertex and needs to be
-   * initialized for the whole mesh when the stroke starts. */
-  if (automasking->factor) {
-    return automasking->factor[vert];
-  }
-
   if (automasking->settings.flags & BRUSH_AUTOMASKING_FACE_SETS) {
     if (!SCULPT_vertex_has_face_set(ss, vert, automasking->settings.initial_face_set)) {
       return 0.0f;
@@ -147,6 +142,21 @@ float SCULPT_automasking_factor_get(AutomaskingCache *automasking, SculptSession
   return 1.0f;
 }
 
+float SCULPT_automasking_factor_get(AutomaskingCache *automasking, SculptSession *ss, int vert)
+{
+  if (!automasking) {
+    return 1.0f;
+  }
+  /* If the cache is initialized with valid info, use the cache. This is used when the
+   * automasking information can't be computed in real time per vertex and needs to be
+   * initialized for the whole mesh when the stroke starts. */
+  if (automasking->factor) {
+    return automasking->factor[vert];
+  }
+
+  return sculpt_automasking_factor_calc(automasking, ss, vert);
+}
+
 void SCULPT_automasking_cache_free(AutomaskingCache *automasking)
 {
   if (!automasking) {
@@ -335,11 +345,23 @@ AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object
   SCULPT_automasking_cache_settings_update(automasking, ss, sd, brush);
   SCULPT_boundary_info_ensure(ob);
 
+  if (!SCULPT_automasking_needs_factors_cache(sd, brush) && ss->cache && ss->cache->use_pixels) {
+    /*
+     * Allocate factor cache but don't initialize it.
+     * Will be filled in by SCULPT_automasking_cache_check.
+     */
+    automasking->factor = (float *)MEM_calloc_arrayN(totvert, sizeof(float), "automask_factor");
+
+    return automasking;
+  }
+
   if (!SCULPT_automasking_needs_factors_cache(sd, brush)) {
     return automasking;
   }
 
+  automasking->has_full_factor_cache = true;
   automasking->factor = (float *)MEM_malloc_arrayN(totvert, sizeof(float), "automask_factor");
+
   for (int i : IndexRange(totvert)) {
     automasking->factor[i] = 1.0f;
   }
@@ -370,3 +392,71 @@ AutomaskingCache *SCULPT_automasking_cache_init(Sculpt *sd, Brush *brush, Object
 
   return automasking;
 }
+
+void SCULPT_automasking_cache_check(SculptSession *ss,
+                                    AutomaskingCache *automasking,
+                                    PBVHNode **nodes,
+                                    int totnode)
+{
+  if (!automasking || automasking->has_full_factor_cache) {
+    return;
+  }
+
+  auto cb = [&](PBVHNode *node) {
+    if (!BKE_pbvh_node_needs_automasking(ss->pbvh, node)) {
+      return;
+    }
+
+    BKE_pbvh_node_automasking_unmark(ss->pbvh, node);
+    PBVHVertexIter vi;
+
+    BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vi, PBVH_ITER_ALL) {
+      if (vi.i >= vi.unique_verts) {
+      }
+      else {
+        automasking->factor[vi.index] = SCULPT_automasking_factor_get(
+            automasking, ss, ss->active_face_index);
+      }
+    }
+    BKE_pbvh_vertex_iter_end;
+  };
+
+  Vector<Vector<int>> node_other_verts;
+  node_other_verts.resize(totnode);
+
+  blender::threading::parallel_for(IndexRange(totnode), 2, [&](IndexRange range) {
+    for (int i : range) {
+      PBVHNode *node = nodes[i];
+
+      if (!BKE_pbvh_node_needs_automasking(ss->pbvh, node)) {
+        return;
+      }
+
+      BKE_pbvh_node_automasking_unmark(ss->pbvh, node);
+      PBVHVertexIter vi;
+
+      BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vi, PBVH_ITER_ALL) {
+        if (vi.i >= vi.unique_verts) {
+          node_other_verts[i].append(vi.index);
+        }
+        else {
+          automasking->factor[vi.index] = sculpt_automasking_factor_calc(
+              automasking, ss, vi.index);
+        }
+      }
+      BKE_pbvh_vertex_iter_end;
+    }
+  });
+
+  Set<int> done_set;
+
+  for (int i : IndexRange(totnode)) {
+    for (int vertex : node_other_verts[i]) {
+      if (!done_set.contains(vertex)) {
+        done_set.add(vertex);
+
+        automasking->factor[vertex] = sculpt_automasking_factor_calc(automasking, ss, vertex);
+      }
+    }
+  }
+}
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index e8a2d35ccd0..861754f946c 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -394,6 +394,9 @@ typedef struct AutomaskingSettings {
   /* Flags from eAutomasking_flag. */
   int flags;
   int initial_face_set;
+  float start_normal_limit, start_normal_falloff;
+  float view_normal_limit, view_normal_falloff;
+  bool use_original_normal;
 } AutomaskingSettings;
 
 typedef struct AutomaskingCache {
@@ -401,6 +404,7 @@ typedef struct AutomaskingCache {
   /* Precomputed auto-mask factor indexed by vertex, owned by the auto-masking system and
    * initialized in #SCULPT_automasking_cache_init when needed. */
   float *factor;
+  bool has_full_factor_cache; /* Cache was built for entire mesh at once. */
 } AutomaskingCache;
 
 typedef struct FilterCache {
@@ -631,6 +635,7 @@ typedef struct StrokeCache {
   rcti previous_r; /* previous redraw rectangle */
   rcti current_r;  /* current redraw rectangle */
 
+  bool use_pixels;
 } StrokeCache;
 
 /* -------------------------------------------------------------------- */
@@ -1288,6 +1293,10 @@ float *SCULPT_boundary_automasking_init(Object *ob,
                                         eBoundaryAutomaskMode mode,
                                         int propagation_steps,
                                         float *automask_factor);
+
+bool SCULPT_automasking_needs_normal(const SculptSession *ss,
+                                     const Sculpt *sculpt,
+                                     const Brush *brush);
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -1805,6 +1814,11 @@ void SCULPT_OT_brush_stroke(struct wmOperatorType *ot);
 
 /* end sculpt_ops.c */
 
+void SCULPT_automasking_cache_check(struct SculptSession *ss,
+                                    struct AutomaskingCache *automasking,
+                                    struct PBVHNode **nodes,
+                                    int totnode);
+
 #define SCULPT_TOOL_NEEDS_COLOR(tool) ELEM(tool, SCULPT_TOOL_PAINT, SCULPT_TOOL_SMEAR)
 
 #ifdef __cplusplus
diff --git a/source/blender/editors/sculpt_paint/sculpt_paint_image.cc b/source/blender/editors/sculpt_paint/sculpt_paint_image.cc
index 975a8f21aaf..6efd8529f1d 100644
--- a/source/blender/editors/sculpt_paint/sculpt_paint_image.cc
+++ b/source/blender/editors/sculpt_paint/sculpt_paint_image.cc
@@ -125,6 +125,21 @@ class ImageBufferByte4 {
   }
 };
 
+static float paint_automasking_interp(SculptSession *ss,
+                                      const TrianglePaintInput &triangle,
+                                      const float2 uv)
+{
+  if (!ss->cache->automasking) {
+    return 1.0f;
+  }
+
+  float a = SCULPT_automasking_factor_get(ss->cache->automasking, ss, triangle.vert_indices[0]);
+  float b = SCULPT_automasking_factor_get(ss->cache->automasking, ss, triangle.vert_indices[1]);
+  float c = SCULPT_automasking_factor_get(ss->cache->automasking, ss, triangle.vert_indices[2]);
+
+  return a * uv[0] + b * uv[1] + c * (1.0 - uv[0] - uv[1]);
+}
+
 template<typename ImageBuffer> class PaintingKernel {
   ImageBuffer image_accessor;
 
@@ -160,9 +175,13 @@ template<typename ImageBuffer> class PaintingKernel {
     float3 pixel_pos = get_start_pixel_pos(triangle, pixel_row);
     const float3 delta_pixel_pos = get_delta_pixel_pos(triangle, pixel_row, pixel_pos);
     bool pixels_painted = false;
+
+    float2 uv = pixel_row.start_barycentric_coord;
+
     for (int x = 0; x < pixel_row.num_pixels; x++) {
       if (!brush_test_fn(&test, pixel_pos)) {
         pixel_pos += delta_pixel_pos;
+        uv += triangle.delta_barycentric_coord_u;
         image_accessor.next_pixel();
         continue;
       }
@@ -171,8 +190,11 @@ template<typename ImageBuffer> class PaintingKernel {
       const float3 normal(0.0f, 0.0f, 0.0f);
       const float3 face_normal(0.0f, 0.0f, 0.0f);
       const float mask = 0.0f;
-      const float falloff_strength = SCULPT_brush_strength_factor(
-          ss, brush, pixel_pos, sqrtf(test.dist), normal, face_normal, mask, 0, thread_id);
+      float falloff_strength = SCULPT_brush_strength_factor(
+          ss, brush, pixel_pos, sqrtf(test.dist), normal, face_normal, mask, -1, thread_id);
+
+      falloff_strength *= paint_automasking_interp(ss, triangle, uv);
+
       float4 paint_color = brush_color * falloff_strength * brush_strength;
       float4 buffer_color;
       blend_color_mix_float(buffer_color, color, paint_color);
@@ -183,6 +205,7 @@ template<typename ImageBuffer> class PaintingKernel {
 
       image_accessor.next_pixel();
       pixel_pos += delta_pixel_pos;
+      uv += triangle.delta_barycentric_coord_u;
     }
     return pixels_painted;
   }
@@ -500,6 +523,8 @@ void SCULPT_do_paint_brush_image(
     return;
   }
 
+  SCULPT_automasking_cache_check(ob->sculpt, ob->sculpt->cache->automasking, nodes, totnode);
+
   TaskParallelSettings settings;
   BKE_pbvh_parallel_range_settings(&settings, true, totnode);
   BLI_task_parallel_range(0, totnode, &data, do_push_undo_tile, &settings);