From ed7f5713f8f9d605e3cd4cce42e40fb5c6bf4bf5 Mon Sep 17 00:00:00 2001 From: Joseph Eagar Date: Mon, 3 Oct 2022 15:33:14 -0700 Subject: Sculpt: Fix T101430: Blank stroke undo steps corrupt dyntopo Blank brush strokes never have an oppurtunity to initialize a sculpt undo step, which results in blank global undo steps. These confuse DynTopo's internal BMLog undo stack. Note: I tried having the stroke operator return OPERATOR_CANCELLED, but this didn't prevent the undo push (which is done automatically due to the presence of OPTYPE_UNDO in the operator flags). I might investigate removing the flag, but that might cause other problems. --- source/blender/editors/sculpt_paint/paint_intern.h | 2 + source/blender/editors/sculpt_paint/paint_stroke.c | 5 ++ source/blender/editors/sculpt_paint/sculpt.c | 80 +++++++++++++++++----- source/blender/editors/sculpt_paint/sculpt_undo.c | 2 +- 4 files changed, 69 insertions(+), 20 deletions(-) (limited to 'source/blender/editors') diff --git a/source/blender/editors/sculpt_paint/paint_intern.h b/source/blender/editors/sculpt_paint/paint_intern.h index 99c25953d50..c6fe7ed3072 100644 --- a/source/blender/editors/sculpt_paint/paint_intern.h +++ b/source/blender/editors/sculpt_paint/paint_intern.h @@ -100,6 +100,8 @@ struct ViewContext *paint_stroke_view_context(struct PaintStroke *stroke); void *paint_stroke_mode_data(struct PaintStroke *stroke); float paint_stroke_distance_get(struct PaintStroke *stroke); void paint_stroke_set_mode_data(struct PaintStroke *stroke, void *mode_data); +bool paint_stroke_started(struct PaintStroke *stroke); + bool PAINT_brush_tool_poll(struct bContext *C); /** * Delete overlay cursor textures to preserve memory and invalidate all overlay flags. diff --git a/source/blender/editors/sculpt_paint/paint_stroke.c b/source/blender/editors/sculpt_paint/paint_stroke.c index 73d52febfc6..bc4c6dc4148 100644 --- a/source/blender/editors/sculpt_paint/paint_stroke.c +++ b/source/blender/editors/sculpt_paint/paint_stroke.c @@ -1674,6 +1674,11 @@ void paint_stroke_set_mode_data(PaintStroke *stroke, void *mode_data) stroke->mode_data = mode_data; } +bool paint_stroke_started(PaintStroke *stroke) +{ + return stroke->stroke_started; +} + bool PAINT_brush_tool_poll(bContext *C) { Paint *p = BKE_paint_get_active_from_context(C); diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c index 3ac0dac137b..c9d29b3ceb5 100644 --- a/source/blender/editors/sculpt_paint/sculpt.c +++ b/source/blender/editors/sculpt_paint/sculpt.c @@ -5354,6 +5354,38 @@ static bool over_mesh(bContext *C, struct wmOperator *UNUSED(op), const float mv return SCULPT_stroke_get_location(C, co_dummy, mval, false); } +static void sculpt_stroke_undo_begin(const bContext *C, wmOperator *op) +{ + Object *ob = CTX_data_active_object(C); + Sculpt *sd = CTX_data_tool_settings(C)->sculpt; + Brush *brush = BKE_paint_brush(&sd->paint); + ToolSettings *tool_settings = CTX_data_tool_settings(C); + + /* Setup the correct undo system. Image painting and sculpting are mutual exclusive. + * Color attributes are part of the sculpting undo system. */ + if (brush && brush->sculpt_tool == SCULPT_TOOL_PAINT && + SCULPT_use_image_paint_brush(&tool_settings->paint_mode, ob)) { + ED_image_undo_push_begin(op->type->name, PAINT_MODE_SCULPT); + } + else { + SCULPT_undo_push_begin_ex(ob, sculpt_tool_name(sd)); + } +} + +static void sculpt_stroke_undo_end(const bContext *C, Brush *brush) +{ + Object *ob = CTX_data_active_object(C); + ToolSettings *tool_settings = CTX_data_tool_settings(C); + + if (brush && brush->sculpt_tool == SCULPT_TOOL_PAINT && + SCULPT_use_image_paint_brush(&tool_settings->paint_mode, ob)) { + ED_image_undo_push_end(); + } + else { + SCULPT_undo_push_end(ob); + } +} + bool SCULPT_handles_colors_report(SculptSession *ss, ReportList *reports) { switch (BKE_pbvh_type(ss->pbvh)) { @@ -5402,15 +5434,7 @@ static bool sculpt_stroke_test_start(bContext *C, struct wmOperator *op, const f SculptCursorGeometryInfo sgi; SCULPT_cursor_geometry_info_update(C, &sgi, mval, false); - /* Setup the correct undo system. Image painting and sculpting are mutual exclusive. - * Color attributes are part of the sculpting undo system. */ - if (brush && brush->sculpt_tool == SCULPT_TOOL_PAINT && - SCULPT_use_image_paint_brush(&tool_settings->paint_mode, ob)) { - ED_image_undo_push_begin(op->type->name, PAINT_MODE_SCULPT); - } - else { - SCULPT_undo_push_begin_ex(ob, sculpt_tool_name(sd)); - } + sculpt_stroke_undo_begin(C, op); SCULPT_stroke_id_next(ob); ss->cache->stroke_id = ss->stroke_id; @@ -5542,13 +5566,7 @@ static void sculpt_stroke_done(const bContext *C, struct PaintStroke *UNUSED(str SCULPT_cache_free(ss->cache); ss->cache = NULL; - if (brush && brush->sculpt_tool == SCULPT_TOOL_PAINT && - SCULPT_use_image_paint_brush(&tool_settings->paint_mode, ob)) { - ED_image_undo_push_end(); - } - else { - SCULPT_undo_push_end(ob); - } + sculpt_stroke_undo_end(C, brush); if (brush->sculpt_tool == SCULPT_TOOL_MASK) { SCULPT_flush_update_done(C, ob, SCULPT_UPDATE_MASK); @@ -5627,9 +5645,10 @@ static int sculpt_brush_stroke_invoke(bContext *C, wmOperator *op, const wmEvent return OPERATOR_PASS_THROUGH; } - if ((retval = op->type->modal(C, op, event)) == OPERATOR_FINISHED) { + retval = op->type->modal(C, op, event); + if (ELEM(retval, OPERATOR_FINISHED, OPERATOR_CANCELLED)) { paint_stroke_free(C, op, op->customdata); - return OPERATOR_FINISHED; + return retval; } /* Add modal handler. */ WM_event_add_modal_handler(C, op); @@ -5684,7 +5703,30 @@ static void sculpt_brush_stroke_cancel(bContext *C, wmOperator *op) static int sculpt_brush_stroke_modal(bContext *C, wmOperator *op, const wmEvent *event) { - return paint_stroke_modal(C, op, event, (struct PaintStroke **)&op->customdata); + bool started = op->customdata && paint_stroke_started((struct PaintStroke *)op->customdata); + + int retval = paint_stroke_modal(C, op, event, (struct PaintStroke **)&op->customdata); + + if (!started && ELEM(retval, OPERATOR_FINISHED, OPERATOR_CANCELLED)) { + /* Did the stroke never start? If so push a blank sculpt undo + * step to prevent a global undo step (which is triggered by the + * OPTYPE_UNDO flag in SCULPT_OT_brush_stroke). + * + * Having blank global undo steps interleaved with sculpt steps + * corrupts the DynTopo undo stack. + * See T101430. + * + * Note: simply returning OPERATOR_CANCELLED was not + * sufficient to prevent this. + */ + Sculpt *sd = CTX_data_tool_settings(C)->sculpt; + Brush *brush = BKE_paint_brush(&sd->paint); + + sculpt_stroke_undo_begin(C, op); + sculpt_stroke_undo_end(C, brush); + } + + return retval; } static void sculpt_redo_empty_ui(bContext *UNUSED(C), wmOperator *UNUSED(op)) diff --git a/source/blender/editors/sculpt_paint/sculpt_undo.c b/source/blender/editors/sculpt_paint/sculpt_undo.c index 2119d33414c..eabaa3c5f4f 100644 --- a/source/blender/editors/sculpt_paint/sculpt_undo.c +++ b/source/blender/editors/sculpt_paint/sculpt_undo.c @@ -83,7 +83,7 @@ #include "sculpt_intern.h" /* Uncomment to print the undo stack in the console on push/undo/redo. */ -#define SCULPT_UNDO_DEBUG +//#define SCULPT_UNDO_DEBUG /* Implementation of undo system for objects in sculpt mode. * -- cgit v1.2.3