Sculpt: Fix T101718: Automasking crash with new texture paint

* 'Original coordinates' mode can't be used in image paint mode
(it does nothing in this case anyway).
* SCULPT_automasking_factor_get now returns 1.0 if
  a PBVH_REF_NONE vertex reference is passed in.

2 files changed, 11 insertions, 2 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index fdbb1f04c0c..129bb604ed0 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -4272,7 +4272,8 @@ static void sculpt_update_cache_invariants(
     bContext *C, Sculpt *sd, SculptSession *ss, wmOperator *op, const float mval[2])
 {
   StrokeCache *cache = MEM_callocN(sizeof(StrokeCache), "stroke cache");
-  UnifiedPaintSettings *ups = &CTX_data_tool_settings(C)->unified_paint_settings;
+  ToolSettings *tool_settings = CTX_data_tool_settings(C);
+  UnifiedPaintSettings *ups = &tool_settings->unified_paint_settings;
   Brush *brush = BKE_paint_brush(&sd->paint);
   ViewContext *vc = paint_stroke_view_context(op->customdata);
   Object *ob = CTX_data_active_object(C);
@@ -4407,6 +4408,14 @@ static void sculpt_update_cache_invariants(
     }
   }
 
+  /* Original coordinates require the sculpt undo system, which isn't used
+   * for image brushes.  It's also not necassary, just disable it.
+   */
+  if (brush && brush->sculpt_tool == SCULPT_TOOL_PAINT &&
+      SCULPT_use_image_paint_brush(&tool_settings->paint_mode, ob)) {
+    cache->original = false;
+  }
+
   cache->first_time = true;
 
 #define PIXEL_INPUT_THRESHHOLD 5
diff --git a/source/blender/editors/sculpt_paint/sculpt_automasking.cc b/source/blender/editors/sculpt_paint/sculpt_automasking.cc
index 5a0fcd8bc27..505440c9272 100644
--- a/source/blender/editors/sculpt_paint/sculpt_automasking.cc
+++ b/source/blender/editors/sculpt_paint/sculpt_automasking.cc
@@ -503,7 +503,7 @@ float SCULPT_automasking_factor_get(AutomaskingCache *automasking,
                                     PBVHVertRef vert,
                                     AutomaskingNodeData *automask_data)
 {
-  if (!automasking) {
+  if (!automasking || vert.i == PBVH_REF_NONE) {
     return 1.0f;
   }