Fix T78242: Crash when using a Sculpt color tools that needs connectivity for the first time

When there is no color layer available,
BKE_sculpt_update_object_for_edit creates a new one and tags the mesh
with ID_RECLAC_GEOMETRY, so this layer is inmediatly available when the
tool starts. This also deletes the PBVH and when it is created again in
BKE_sculpt_update_object_after_eval, the pmap is not initialized, making
the tool crash.

This moves the color layer creation to a separate function outside
BKE_sculpt_update_object_for_edit, which now runs after the color
layer is available, so it won't need to update again and the pmap will
still be available when the tool is used.

Reviewed By: sergey

Maniphest Tasks: T78242

Differential Revision: https://developer.blender.org/D8135

2 files changed, 22 insertions, 12 deletions

diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h
index e3a6fb4ba2e..1c02aece06f 100644
--- a/source/blender/blenkernel/BKE_paint.h
+++ b/source/blender/blenkernel/BKE_paint.h
@@ -451,6 +451,10 @@ void BKE_sculptsession_free_vwpaint_data(struct SculptSession *ss);
 void BKE_sculptsession_bm_to_me(struct Object *ob, bool reorder);
 void BKE_sculptsession_bm_to_me_for_render(struct Object *object);
 
+/* Create new color layer on object if it doesn't have one and if experimental feature set has
+ * sculpt vertex color enabled. Returns truth if new layer has been added, false otherwise. */
+void BKE_sculpt_color_layer_create_if_needed(struct Object *object);
+
 void BKE_sculpt_update_object_for_edit(struct Depsgraph *depsgraph,
                                        struct Object *ob_orig,
                                        bool need_pmap,
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index 50feb8e99c8..5feaacee254 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -1484,7 +1484,7 @@ static void sculpt_update_object(Depsgraph *depsgraph,
                                  Mesh *me_eval,
                                  bool need_pmap,
                                  bool need_mask,
-                                 bool need_colors)
+                                 bool UNUSED(need_colors))
 {
   Scene *scene = DEG_get_input_scene(depsgraph);
   Sculpt *sd = scene->toolsettings->sculpt;
@@ -1514,16 +1514,6 @@ static void sculpt_update_object(Depsgraph *depsgraph,
     }
   }
 
-  /* Add a color layer if a color tool is used. */
-  Mesh *orig_me = BKE_object_get_original_mesh(ob);
-  if (need_colors && U.experimental.use_sculpt_vertex_colors) {
-    if (!CustomData_has_layer(&orig_me->vdata, CD_PROP_COLOR)) {
-      CustomData_add_layer(&orig_me->vdata, CD_PROP_COLOR, CD_DEFAULT, NULL, orig_me->totvert);
-      BKE_mesh_update_customdata_pointers(orig_me, true);
-      DEG_id_tag_update(&orig_me->id, ID_RECALC_GEOMETRY);
-    }
-  }
-
   /* tessfaces aren't used and will become invalid */
   BKE_mesh_tessface_clear(me);
 
@@ -1684,10 +1674,26 @@ void BKE_sculpt_update_object_after_eval(Depsgraph *depsgraph, Object *ob_eval)
   Mesh *me_eval = BKE_object_get_evaluated_mesh(ob_eval);
 
   BLI_assert(me_eval != NULL);
-
   sculpt_update_object(depsgraph, ob_orig, me_eval, false, false, false);
 }
 
+void BKE_sculpt_color_layer_create_if_needed(struct Object *object)
+{
+  Mesh *orig_me = BKE_object_get_original_mesh(object);
+  if (!U.experimental.use_sculpt_vertex_colors) {
+    return;
+  }
+
+  if (CustomData_has_layer(&orig_me->vdata, CD_PROP_COLOR)) {
+    return;
+  }
+
+  CustomData_add_layer(&orig_me->vdata, CD_PROP_COLOR, CD_DEFAULT, NULL, orig_me->totvert);
+  BKE_mesh_update_customdata_pointers(orig_me, true);
+  DEG_id_tag_update(&orig_me->id, ID_RECALC_GEOMETRY);
+  return;
+}
+
 void BKE_sculpt_update_object_for_edit(
     Depsgraph *depsgraph, Object *ob_orig, bool need_pmap, bool need_mask, bool need_colors)
 {