Undo System: remove accumulate/store modes

This complicated handling of undo steps in a generic way
especially switching between undo systems that stored data to ones
that accumulated changes.

Now each undo system must treat it's steps as check-point,
internally it can apply/rewind changes.

This commit also fixes projection paint where the object mode wasn't
following the undo steps.

5 files changed, 172 insertions, 38 deletions

diff --git a/source/blender/editors/sculpt_paint/paint_curve_undo.c b/source/blender/editors/sculpt_paint/paint_curve_undo.c
index fbd2d0153a1..02d95ed54f8 100644
--- a/source/blender/editors/sculpt_paint/paint_curve_undo.c
+++ b/source/blender/editors/sculpt_paint/paint_curve_undo.c
@@ -134,7 +134,6 @@ void ED_paintcurve_undosys_type(UndoType *ut)
 	ut->step_decode = paintcurve_undosys_step_decode;
 	ut->step_free = paintcurve_undosys_step_free;
 
-	ut->mode = BKE_UNDOTYPE_MODE_STORE;
 	ut->use_context = false;
 
 	ut->step_size = sizeof(PaintCurveUndoStep);
diff --git a/source/blender/editors/sculpt_paint/paint_image.c b/source/blender/editors/sculpt_paint/paint_image.c
index 8bd83900439..92b00a1505f 100644
--- a/source/blender/editors/sculpt_paint/paint_image.c
+++ b/source/blender/editors/sculpt_paint/paint_image.c
@@ -489,7 +489,7 @@ static PaintOperation *texture_paint_init(bContext *C, wmOperator *op, const flo
 	}
 
 	settings->imapaint.flag |= IMAGEPAINT_DRAWING;
-	ED_image_undo_push_begin(op->type->name);
+	ED_image_undo_push_begin(op->type->name, PAINT_MODE_TEXTURE_2D);
 
 	return pop;
 }
@@ -1234,7 +1234,7 @@ void ED_imapaint_bucket_fill(struct bContext *C, float color[3], wmOperator *op)
 
 	BKE_undosys_step_push_init_with_type(wm->undo_stack, C, op->type->name, BKE_UNDOSYS_TYPE_IMAGE);
 
-	ED_image_undo_push_begin(op->type->name);
+	ED_image_undo_push_begin(op->type->name, PAINT_MODE_TEXTURE_2D);
 
 	paint_2d_bucket_fill(C, color, NULL, NULL, NULL);
 
diff --git a/source/blender/editors/sculpt_paint/paint_image_proj.c b/source/blender/editors/sculpt_paint/paint_image_proj.c
index 182a0912533..bb9391a052c 100644
--- a/source/blender/editors/sculpt_paint/paint_image_proj.c
+++ b/source/blender/editors/sculpt_paint/paint_image_proj.c
@@ -5546,7 +5546,7 @@ static int texture_paint_camera_project_exec(bContext *C, wmOperator *op)
 
 	scene->toolsettings->imapaint.flag |= IMAGEPAINT_DRAWING;
 
-	ED_image_undo_push_begin(op->type->name);
+	ED_image_undo_push_begin(op->type->name, PAINT_MODE_TEXTURE_3D);
 
 	/* allocate and initialize spatial data structures */
 	project_paint_begin(C, &ps, false, 0);
diff --git a/source/blender/editors/sculpt_paint/paint_image_undo.c b/source/blender/editors/sculpt_paint/paint_image_undo.c
index 04d58877608..20696ed1e91 100644
--- a/source/blender/editors/sculpt_paint/paint_image_undo.c
+++ b/source/blender/editors/sculpt_paint/paint_image_undo.c
@@ -36,6 +36,7 @@
 
 #include "BKE_context.h"
 #include "BKE_image.h"
+#include "BKE_paint.h"
 #include "BKE_undo_system.h"
 
 #include "DEG_depsgraph.h"
@@ -43,6 +44,7 @@
 #include "ED_paint.h"
 #include "ED_undo.h"
 #include "ED_util.h"
+#include "ED_object.h"
 
 #include "GPU_draw.h"
 
@@ -349,19 +351,6 @@ static void image_undo_free_list(ListBase *lb)
 	}
 }
 
-void ED_image_undo_push_begin(const char *name)
-{
-	UndoStack *ustack = ED_undo_stack_get();
-	bContext *C = NULL; /* special case, we never read from this. */
-	BKE_undosys_step_push_init_with_type(ustack, C, name, BKE_UNDOSYS_TYPE_IMAGE);
-}
-
-void ED_image_undo_push_end(void)
-{
-	UndoStack *ustack = ED_undo_stack_get();
-	BKE_undosys_step_push(ustack, NULL, NULL);
-}
-
 static void image_undo_invalidate(void)
 {
 	UndoImageTile *tile;
@@ -381,6 +370,8 @@ static void image_undo_invalidate(void)
 typedef struct ImageUndoStep {
 	UndoStep step;
 	ListBase tiles;
+	bool is_encode_init;
+	ePaintMode paint_mode;
 
 	/* Use for all ID lookups (can be NULL). */
 	struct UndoIDPtrMap *id_map;
@@ -432,10 +423,11 @@ static void image_undosys_step_encode_init(struct bContext *UNUSED(C), UndoStep
 {
 	ImageUndoStep *us = (ImageUndoStep *)us_p;
 	/* dummy, memory is cleared anyway. */
+	us->is_encode_init = true;
 	BLI_listbase_clear(&us->tiles);
 }
 
-static bool image_undosys_step_encode(struct bContext *UNUSED(C), struct Main *UNUSED(bmain), UndoStep *us_p)
+static bool image_undosys_step_encode(struct bContext *C, struct Main *UNUSED(bmain), UndoStep *us_p)
 {
 	/* dummy, encoding is done along the way by adding tiles
 	 * to the current 'ImageUndoStep' added by encode_init. */
@@ -445,33 +437,100 @@ static bool image_undosys_step_encode(struct bContext *UNUSED(C), struct Main *U
 
 	int allocsize = IMAPAINT_TILE_SIZE * IMAPAINT_TILE_SIZE * 4;
 
-
-	/* first dispose of invalid tiles (may happen due to drag dot for instance) */
-	for (UndoImageTile *tile = us->tiles.first; tile;) {
-		if (!tile->valid) {
-			UndoImageTile *tmp_tile = tile->next;
-			MEM_freeN(tile->rect.pt);
-			BLI_freelinkN(&us->tiles, tile);
-			tile = tmp_tile;
-		}
-		else {
-			us->step.data_size += allocsize * ((tile->use_float) ? sizeof(float) : sizeof(char));
-			tile = tile->next;
+	if (us->is_encode_init) {
+		/* first dispose of invalid tiles (may happen due to drag dot for instance) */
+		for (UndoImageTile *tile = us->tiles.first; tile;) {
+			if (!tile->valid) {
+				UndoImageTile *tmp_tile = tile->next;
+				MEM_freeN(tile->rect.pt);
+				BLI_freelinkN(&us->tiles, tile);
+				tile = tmp_tile;
+			}
+			else {
+				us->step.data_size += allocsize * ((tile->use_float) ? sizeof(float) : sizeof(char));
+				tile = tile->next;
+			}
 		}
 	}
+	else {
+		/* Happens when switching modes. */
+		ePaintMode paint_mode = BKE_paintmode_get_active_from_context(C);
+		BLI_assert(ELEM(paint_mode, PAINT_MODE_TEXTURE_2D, PAINT_MODE_TEXTURE_3D));
+		us->paint_mode = paint_mode;
+	}
 
 	image_undosys_step_encode_store_ids(us);
 
+	us_p->is_applied = true;
+
 	return true;
 }
 
-static void image_undosys_step_decode(struct bContext *UNUSED(C), struct Main *bmain, UndoStep *us_p, int UNUSED(dir))
+
+static void image_undosys_step_decode_undo_impl(ImageUndoStep *us)
+{
+	BLI_assert(us->step.is_applied == true);
+	image_undo_restore_list(&us->tiles, us->id_map);
+	us->step.is_applied = false;
+}
+
+static void image_undosys_step_decode_redo_impl(ImageUndoStep *us)
+{
+	BLI_assert(us->step.is_applied == false);
+	image_undo_restore_list(&us->tiles, us->id_map);
+	us->step.is_applied = true;
+}
+
+static void image_undosys_step_decode_undo(ImageUndoStep *us)
+{
+	ImageUndoStep *us_iter = us;
+	while (us_iter->step.next && (us_iter->step.next->type == us_iter->step.type)) {
+		if (us_iter->step.next->is_applied == false) {
+			break;
+		}
+		us_iter = (ImageUndoStep *)us_iter->step.next;
+	}
+	while (us_iter != us) {
+		image_undosys_step_decode_undo_impl(us_iter);
+		us_iter = (ImageUndoStep *)us_iter->step.prev;
+	}
+}
+
+static void image_undosys_step_decode_redo(ImageUndoStep *us)
+{
+	ImageUndoStep *us_iter = us;
+	while (us_iter->step.prev && (us_iter->step.prev->type == us_iter->step.type)) {
+		if (us_iter->step.prev->is_applied == true) {
+			break;
+		}
+		us_iter = (ImageUndoStep *)us_iter->step.prev;
+	}
+	while (us_iter && (us_iter->step.is_applied == false)) {
+		image_undosys_step_decode_redo_impl(us_iter);
+		if (us_iter == us) {
+			break;
+		}
+		us_iter = (ImageUndoStep *)us_iter->step.next;
+	}
+}
+
+static void image_undosys_step_decode(struct bContext *C, struct Main *bmain, UndoStep *us_p, int dir)
 {
 	ImageUndoStep *us = (ImageUndoStep *)us_p;
 #if 0
 	paint_undosys_step_decode_restore_ids(us);
 #endif
-	image_undo_restore_list(&us->tiles, us->id_map);
+
+	if (dir < 0) {
+		image_undosys_step_decode_undo(us);
+	}
+	else {
+		image_undosys_step_decode_redo(us);
+	}
+
+	if (us->paint_mode == PAINT_MODE_TEXTURE_3D) {
+		ED_object_mode_set(C, OB_MODE_TEXTURE_PAINT);
+	}
 
 	/* Refresh texture slots. */
 	ED_editors_init_for_undo(bmain);
@@ -505,7 +564,6 @@ void ED_image_undosys_type(UndoType *ut)
 
 	ut->step_foreach_ID_ref = image_undosys_foreach_ID_ref;
 
-	ut->mode = BKE_UNDOTYPE_MODE_ACCUMULATE;
 	ut->use_context = true;
 
 	ut->step_size = sizeof(ImageUndoStep);
@@ -527,8 +585,17 @@ ListBase *ED_image_undosys_step_get_tiles(UndoStep *us_p)
 ListBase *ED_image_undo_get_tiles(void)
 {
 	UndoStack *ustack = ED_undo_stack_get();
-	UndoStep *us = BKE_undosys_stack_init_or_active_with_type(ustack, BKE_UNDOSYS_TYPE_IMAGE);
-	return ED_image_undosys_step_get_tiles(us);
+	UndoStep *us_prev = ustack->step_init;
+	UndoStep *us_p = BKE_undosys_stack_init_or_active_with_type(ustack, BKE_UNDOSYS_TYPE_IMAGE);
+	ImageUndoStep *us = (ImageUndoStep *)us_p;
+	/* We should always have an undo push started when accessing tiles,
+	 * not doing this means we won't have paint_mode correctly set. */
+	BLI_assert(us_p == us_prev);
+	if (us_p != us_prev) {
+		/* Fallback value until we can be sure this never happens. */
+		us->paint_mode = PAINT_MODE_TEXTURE_2D;
+	}
+	return ED_image_undosys_step_get_tiles(us_p);
 }
 
 /* restore painting image to previous state. Used for anchored and drag-dot style brushes*/
@@ -539,4 +606,20 @@ void ED_image_undo_restore(UndoStep *us)
 	image_undo_invalidate();
 }
 
+void ED_image_undo_push_begin(const char *name, int paint_mode)
+{
+	UndoStack *ustack = ED_undo_stack_get();
+	bContext *C = NULL; /* special case, we never read from this. */
+	UndoStep *us_p = BKE_undosys_step_push_init_with_type(ustack, C, name, BKE_UNDOSYS_TYPE_IMAGE);
+	ImageUndoStep *us = (ImageUndoStep *)us_p;
+	BLI_assert(ELEM(paint_mode, PAINT_MODE_TEXTURE_2D, PAINT_MODE_TEXTURE_3D));
+	us->paint_mode = paint_mode;
+}
+
+void ED_image_undo_push_end(void)
+{
+	UndoStack *ustack = ED_undo_stack_get();
+	BKE_undosys_step_push(ustack, NULL, NULL);
+}
+
 /** \} */
diff --git a/source/blender/editors/sculpt_paint/sculpt_undo.c b/source/blender/editors/sculpt_paint/sculpt_undo.c
index 1794fbfe41f..a8c8490fb79 100644
--- a/source/blender/editors/sculpt_paint/sculpt_undo.c
+++ b/source/blender/editors/sculpt_paint/sculpt_undo.c
@@ -1048,17 +1048,70 @@ static bool sculpt_undosys_step_encode(struct bContext *UNUSED(C), struct Main *
 	if (unode && unode->type == SCULPT_UNDO_DYNTOPO_END) {
 		us->step.use_memfile_step = true;
 	}
+	us->step.is_applied = true;
 	return true;
 }
 
-static void sculpt_undosys_step_decode(struct bContext *C, struct Main *UNUSED(bmain), UndoStep *us_p, int UNUSED(dir))
+static void sculpt_undosys_step_decode_undo_impl(struct bContext *C, SculptUndoStep *us)
+{
+	BLI_assert(us->step.is_applied == true);
+	sculpt_undo_restore_list(C, &us->data.nodes);
+	us->step.is_applied = false;
+}
+
+static void sculpt_undosys_step_decode_redo_impl(struct bContext *C, SculptUndoStep *us)
+{
+	BLI_assert(us->step.is_applied == false);
+	sculpt_undo_restore_list(C, &us->data.nodes);
+	us->step.is_applied = true;
+}
+
+static void sculpt_undosys_step_decode_undo(struct bContext *C, SculptUndoStep *us)
+{
+	SculptUndoStep *us_iter = us;
+	while (us_iter->step.next && (us_iter->step.next->type == us_iter->step.type)) {
+		if (us_iter->step.next->is_applied == false) {
+			break;
+		}
+		us_iter = (SculptUndoStep *)us_iter->step.next;
+	}
+	while (us_iter != us) {
+		sculpt_undosys_step_decode_undo_impl(C, us_iter);
+		us_iter = (SculptUndoStep *)us_iter->step.prev;
+	}
+}
+
+static void sculpt_undosys_step_decode_redo(struct bContext *C, SculptUndoStep *us)
+{
+	SculptUndoStep *us_iter = us;
+	while (us_iter->step.prev && (us_iter->step.prev->type == us_iter->step.type)) {
+		if (us_iter->step.prev->is_applied == true) {
+			break;
+		}
+		us_iter = (SculptUndoStep *)us_iter->step.prev;
+	}
+	while (us_iter && (us_iter->step.is_applied == false)) {
+		sculpt_undosys_step_decode_redo_impl(C, us_iter);
+		if (us_iter == us) {
+			break;
+		}
+		us_iter = (SculptUndoStep *)us_iter->step.next;
+	}
+}
+
+static void sculpt_undosys_step_decode(struct bContext *C, struct Main *UNUSED(bmain), UndoStep *us_p, int dir)
 {
 	/* TODO(campbell): undo_system: use low-level API to set mode. */
 	ED_object_mode_set(C, OB_MODE_SCULPT);
 	BLI_assert(sculpt_undosys_poll(C));
 
 	SculptUndoStep *us = (SculptUndoStep *)us_p;
-	sculpt_undo_restore_list(C, &us->data.nodes);
+	if (dir < 0) {
+		sculpt_undosys_step_decode_undo(C, us);
+	}
+	else {
+		sculpt_undosys_step_decode_redo(C, us);
+	}
 }
 
 static void sculpt_undosys_step_free(UndoStep *us_p)
@@ -1077,7 +1130,6 @@ void ED_sculpt_undosys_type(UndoType *ut)
 	ut->step_decode = sculpt_undosys_step_decode;
 	ut->step_free = sculpt_undosys_step_free;
 
-	ut->mode = BKE_UNDOTYPE_MODE_ACCUMULATE;
 	ut->use_context = true;
 
 	ut->step_size = sizeof(SculptUndoStep);