Fix T97469: Sculpt colors crash in multiresolution or dynamic topology modes.

Sculpt paint tools now pop up an error message if
dynamic topology or multires are enabled.

Implementation notes:

* SCULPT_vertex_colors_poll is now a static function in sculpt_ops.c.
  It is now used solely by the legacy color attribute conversion
  operators (SCULPT_OT_vertex_to_loop_colors and SCULPT_OT_loop_to_vertex_colors)
  and should be deleted when they are.
* There is a new method, SCULPT_handles_colors_report, that returns true if
  the sculpt session can handle color attributes; otherwise it returns false
  and displays an error message to the user.

4 files changed, 66 insertions, 23 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index ff13d755971..62f672ed383 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -3914,16 +3914,6 @@ bool SCULPT_mode_poll(bContext *C)
   return ob && ob->mode & OB_MODE_SCULPT;
 }
 
-bool SCULPT_vertex_colors_poll(bContext *C)
-{
-  if (!SCULPT_mode_poll(C)) {
-    return false;
-  }
-
-  Object *ob = CTX_data_active_object(C);
-  return ob->sculpt && SCULPT_has_colors(ob->sculpt);
-}
-
 bool SCULPT_mode_poll_view3d(bContext *C)
 {
   return (SCULPT_mode_poll(C) && CTX_wm_region_view3d(C));
@@ -5256,6 +5246,24 @@ static bool over_mesh(bContext *C, struct wmOperator *UNUSED(op), float x, float
   return SCULPT_stroke_get_location(C, co, mouse);
 }
 
+bool SCULPT_handles_colors_report(SculptSession *ss, ReportList *reports)
+{
+  switch (BKE_pbvh_type(ss->pbvh)) {
+    case PBVH_FACES:
+      return true;
+    case PBVH_BMESH:
+      BKE_report(reports, RPT_ERROR, "Not supported in dynamic topology mode.");
+      return false;
+    case PBVH_GRIDS:
+      BKE_report(reports, RPT_ERROR, "Not supported in multiresolution mode.");
+      return false;
+  }
+
+  BLI_assert_msg(0, "PBVH corruption, type was invalid.");
+
+  return false;
+}
+
 static bool sculpt_stroke_test_start(bContext *C, struct wmOperator *op, const float mouse[2])
 {
   /* Don't start the stroke until mouse goes over the mesh.
@@ -5438,6 +5446,12 @@ static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, const wmEvent
 
   sculpt_brush_stroke_init(C, op);
 
+  Object *ob = CTX_data_active_object(C);
+
+  if (!SCULPT_handles_colors_report(ob->sculpt, op->reports)) {
+    return OPERATOR_CANCELLED;
+  }
+
   stroke = paint_stroke_new(C,
                             op,
                             SCULPT_stroke_get_location,
diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_color.c b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
index 5d4a2c54832..a705f6aa8a8 100644
--- a/source/blender/editors/sculpt_paint/sculpt_filter_color.c
+++ b/source/blender/editors/sculpt_paint/sculpt_filter_color.c
@@ -24,6 +24,7 @@
 #include "BKE_object.h"
 #include "BKE_paint.h"
 #include "BKE_pbvh.h"
+#include "BKE_report.h"
 #include "BKE_scene.h"
 
 #include "IMB_colormanagement.h"
@@ -342,10 +343,7 @@ static int sculpt_color_filter_invoke(bContext *C, wmOperator *op, const wmEvent
   }
 
   /* Disable for multires and dyntopo for now */
-  if (!ss->pbvh) {
-    return OPERATOR_CANCELLED;
-  }
-  if (BKE_pbvh_type(pbvh) != PBVH_FACES) {
+  if (!ss->pbvh || !SCULPT_handles_colors_report(ss, op->reports)) {
     return OPERATOR_CANCELLED;
   }
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_intern.h b/source/blender/editors/sculpt_paint/sculpt_intern.h
index bd95bb4666b..f13f1c79a7a 100644
--- a/source/blender/editors/sculpt_paint/sculpt_intern.h
+++ b/source/blender/editors/sculpt_paint/sculpt_intern.h
@@ -782,7 +782,17 @@ bool SCULPT_mode_poll_view3d(struct bContext *C);
 bool SCULPT_poll(struct bContext *C);
 bool SCULPT_poll_view3d(struct bContext *C);
 
-bool SCULPT_vertex_colors_poll(struct bContext *C);
+/**
+ * Returns true if sculpt session can handle color attributes
+ * (BKE_pbvh_type(ss->pbvh) == PBVH_FACES).  If false an error
+ * message will be shown to the user.  Operators should return
+ * OPERATOR_CANCELLED in this case.
+ *
+ * NOTE: Does not check if a color attribute actually exists.
+ * Calling code must handle this itself; in most cases a call to
+ * BKE_sculpt_color_layer_create_if_needed() is sufficient.
+ */
+bool SCULPT_handles_colors_report(struct SculptSession *ss, struct ReportList *reports);
 
 /** \} */
 
diff --git a/source/blender/editors/sculpt_paint/sculpt_ops.c b/source/blender/editors/sculpt_paint/sculpt_ops.c
index 1984678bb00..4755cf6e9ce 100644
--- a/source/blender/editors/sculpt_paint/sculpt_ops.c
+++ b/source/blender/editors/sculpt_paint/sculpt_ops.c
@@ -662,6 +662,21 @@ static int vertex_to_loop_colors_exec(bContext *C, wmOperator *UNUSED(op))
   return OPERATOR_FINISHED;
 }
 
+static bool sculpt_colors_poll(bContext *C)
+{
+  if (!SCULPT_mode_poll(C)) {
+    return false;
+  }
+
+  Object *ob = CTX_data_active_object(C);
+
+  if (!ob->sculpt || !ob->sculpt->pbvh || BKE_pbvh_type(ob->sculpt->pbvh) != PBVH_FACES) {
+    return false;
+  }
+
+  return SCULPT_has_colors(ob->sculpt);
+}
+
 static void SCULPT_OT_vertex_to_loop_colors(wmOperatorType *ot)
 {
   /* identifiers */
@@ -670,7 +685,7 @@ static void SCULPT_OT_vertex_to_loop_colors(wmOperatorType *ot)
   ot->idname = "SCULPT_OT_vertex_to_loop_colors";
 
   /* api callbacks */
-  ot->poll = SCULPT_vertex_colors_poll;
+  ot->poll = sculpt_colors_poll;
   ot->exec = vertex_to_loop_colors_exec;
 
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
@@ -734,14 +749,14 @@ static void SCULPT_OT_loop_to_vertex_colors(wmOperatorType *ot)
   ot->idname = "SCULPT_OT_loop_to_vertex_colors";
 
   /* api callbacks */
-  ot->poll = SCULPT_vertex_colors_poll;
+  ot->poll = sculpt_colors_poll;
   ot->exec = loop_to_vertex_colors_exec;
 
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
 static int sculpt_sample_color_invoke(bContext *C,
-                                      wmOperator *UNUSED(op),
+                                      wmOperator *op,
                                       const wmEvent *UNUSED(e))
 {
   Sculpt *sd = CTX_data_tool_settings(C)->sculpt;
@@ -752,11 +767,17 @@ static int sculpt_sample_color_invoke(bContext *C,
   int active_vertex = SCULPT_active_vertex_get(ss);
   float active_vertex_color[4];
 
-  if (!SCULPT_has_colors(ss)) {
+  if (!SCULPT_handles_colors_report(ss, op->reports)) {
     return OPERATOR_CANCELLED;
   }
 
-  SCULPT_vertex_color_get(ss, active_vertex, active_vertex_color);
+  /* No color attribute? Set color to white. */
+  if (!SCULPT_has_colors(ss)) {
+    copy_v4_fl(active_vertex_color, 1.0f);
+  }
+  else {
+    SCULPT_vertex_color_get(ss, active_vertex, active_vertex_color);
+  }
 
   float color_srgb[3];
   copy_v3_v3(color_srgb, active_vertex_color);
@@ -777,7 +798,7 @@ static void SCULPT_OT_sample_color(wmOperatorType *ot)
 
   /* api callbacks */
   ot->invoke = sculpt_sample_color_invoke;
-  ot->poll = SCULPT_vertex_colors_poll;
+  ot->poll = SCULPT_mode_poll;
 
   ot->flag = OPTYPE_REGISTER;
 }
@@ -1025,8 +1046,8 @@ static int sculpt_mask_by_color_invoke(bContext *C, wmOperator *op, const wmEven
 
   BKE_sculpt_update_object_for_edit(depsgraph, ob, true, true, false);
 
-  /* Color data is not available in Multires. */
-  if (BKE_pbvh_type(ss->pbvh) != PBVH_FACES) {
+  /* Color data is not available in Multires or dyanmic topology. */
+  if (!SCULPT_handles_colors_report(ss, op->reports)) {
     return OPERATOR_CANCELLED;
   }