From 4f3233dd536e8839c304dd34cd8aec89bfd8ca34 Mon Sep 17 00:00:00 2001 From: Pablo Dobarro Date: Fri, 17 Jul 2020 18:07:47 +0200 Subject: Fix T78242: Crash when using a Sculpt color tools that needs connectivity for the first time When there is no color layer available, BKE_sculpt_update_object_for_edit creates a new one and tags the mesh with ID_RECLAC_GEOMETRY, so this layer is inmediatly available when the tool starts. This also deletes the PBVH and when it is created again in BKE_sculpt_update_object_after_eval, the pmap is not initialized, making the tool crash. This moves the color layer creation to a separate function outside BKE_sculpt_update_object_for_edit, which now runs after the color layer is available, so it won't need to update again and the pmap will still be available when the tool is used. Reviewed By: sergey Maniphest Tasks: T78242 Differential Revision: https://developer.blender.org/D8135 --- source/blender/blenkernel/BKE_paint.h | 4 +++ source/blender/blenkernel/intern/paint.c | 30 +++++++++++++--------- source/blender/editors/sculpt_paint/sculpt.c | 9 ++++++- .../editors/sculpt_paint/sculpt_filter_color.c | 6 ++++- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/source/blender/blenkernel/BKE_paint.h b/source/blender/blenkernel/BKE_paint.h index e3a6fb4ba2e..1c02aece06f 100644 --- a/source/blender/blenkernel/BKE_paint.h +++ b/source/blender/blenkernel/BKE_paint.h @@ -451,6 +451,10 @@ void BKE_sculptsession_free_vwpaint_data(struct SculptSession *ss); void BKE_sculptsession_bm_to_me(struct Object *ob, bool reorder); void BKE_sculptsession_bm_to_me_for_render(struct Object *object); +/* Create new color layer on object if it doesn't have one and if experimental feature set has + * sculpt vertex color enabled. Returns truth if new layer has been added, false otherwise. */ +void BKE_sculpt_color_layer_create_if_needed(struct Object *object); + void BKE_sculpt_update_object_for_edit(struct Depsgraph *depsgraph, struct Object *ob_orig, bool need_pmap, diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c index 50feb8e99c8..5feaacee254 100644 --- a/source/blender/blenkernel/intern/paint.c +++ b/source/blender/blenkernel/intern/paint.c @@ -1484,7 +1484,7 @@ static void sculpt_update_object(Depsgraph *depsgraph, Mesh *me_eval, bool need_pmap, bool need_mask, - bool need_colors) + bool UNUSED(need_colors)) { Scene *scene = DEG_get_input_scene(depsgraph); Sculpt *sd = scene->toolsettings->sculpt; @@ -1514,16 +1514,6 @@ static void sculpt_update_object(Depsgraph *depsgraph, } } - /* Add a color layer if a color tool is used. */ - Mesh *orig_me = BKE_object_get_original_mesh(ob); - if (need_colors && U.experimental.use_sculpt_vertex_colors) { - if (!CustomData_has_layer(&orig_me->vdata, CD_PROP_COLOR)) { - CustomData_add_layer(&orig_me->vdata, CD_PROP_COLOR, CD_DEFAULT, NULL, orig_me->totvert); - BKE_mesh_update_customdata_pointers(orig_me, true); - DEG_id_tag_update(&orig_me->id, ID_RECALC_GEOMETRY); - } - } - /* tessfaces aren't used and will become invalid */ BKE_mesh_tessface_clear(me); @@ -1684,10 +1674,26 @@ void BKE_sculpt_update_object_after_eval(Depsgraph *depsgraph, Object *ob_eval) Mesh *me_eval = BKE_object_get_evaluated_mesh(ob_eval); BLI_assert(me_eval != NULL); - sculpt_update_object(depsgraph, ob_orig, me_eval, false, false, false); } +void BKE_sculpt_color_layer_create_if_needed(struct Object *object) +{ + Mesh *orig_me = BKE_object_get_original_mesh(object); + if (!U.experimental.use_sculpt_vertex_colors) { + return; + } + + if (CustomData_has_layer(&orig_me->vdata, CD_PROP_COLOR)) { + return; + } + + CustomData_add_layer(&orig_me->vdata, CD_PROP_COLOR, CD_DEFAULT, NULL, orig_me->totvert); + BKE_mesh_update_customdata_pointers(orig_me, true); + DEG_id_tag_update(&orig_me->id, ID_RECALC_GEOMETRY); + return; +} + void BKE_sculpt_update_object_for_edit( Depsgraph *depsgraph, Object *ob_orig, bool need_pmap, bool need_mask, bool need_colors) { diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c index da5d6588dc8..aa8161836f9 100644 --- a/source/blender/editors/sculpt_paint/sculpt.c +++ b/source/blender/editors/sculpt_paint/sculpt.c @@ -7140,7 +7140,6 @@ static void sculpt_brush_init_tex(const Scene *scene, Sculpt *sd, SculptSession static void sculpt_brush_stroke_init(bContext *C, wmOperator *op) { - Depsgraph *depsgraph = CTX_data_ensure_evaluated_depsgraph(C); Scene *scene = CTX_data_scene(C); Object *ob = CTX_data_active_object(C); Sculpt *sd = CTX_data_tool_settings(C)->sculpt; @@ -7163,6 +7162,14 @@ static void sculpt_brush_stroke_init(bContext *C, wmOperator *op) is_smooth = sculpt_needs_connectivity_info(sd, brush, ss, mode); needs_colors = ELEM(brush->sculpt_tool, SCULPT_TOOL_PAINT, SCULPT_TOOL_SMEAR); + + if (needs_colors) { + BKE_sculpt_color_layer_create_if_needed(ob); + } + + /* CTX_data_ensure_evaluated_depsgraph should be used at the end to include the updates of + * earlier steps modifying the data. */ + Depsgraph *depsgraph = CTX_data_ensure_evaluated_depsgraph(C); BKE_sculpt_update_object_for_edit(depsgraph, ob, is_smooth, need_mask, needs_colors); } diff --git a/source/blender/editors/sculpt_paint/sculpt_filter_color.c b/source/blender/editors/sculpt_paint/sculpt_filter_color.c index 556b73b0ea5..59d82825740 100644 --- a/source/blender/editors/sculpt_paint/sculpt_filter_color.c +++ b/source/blender/editors/sculpt_paint/sculpt_filter_color.c @@ -265,7 +265,6 @@ static int sculpt_color_filter_modal(bContext *C, wmOperator *op, const wmEvent static int sculpt_color_filter_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event)) { Object *ob = CTX_data_active_object(C); - Depsgraph *depsgraph = CTX_data_depsgraph_pointer(C); Sculpt *sd = CTX_data_tool_settings(C)->sculpt; SculptSession *ss = ob->sculpt; int mode = RNA_enum_get(op->ptr, "type"); @@ -285,6 +284,11 @@ static int sculpt_color_filter_invoke(bContext *C, wmOperator *op, const wmEvent SCULPT_undo_push_begin("color filter"); + BKE_sculpt_color_layer_create_if_needed(ob); + + /* CTX_data_ensure_evaluated_depsgraph should be used at the end to include the updates of + * earlier steps modifying the data. */ + Depsgraph *depsgraph = CTX_data_ensure_evaluated_depsgraph(C); bool needs_pmap = mode == COLOR_FILTER_SMOOTH; BKE_sculpt_update_object_for_edit(depsgraph, ob, needs_pmap, false, true); -- cgit v1.2.3