Workaround T61948: Undo steps created for non-paint operations

Changing the brush size for example,
was adding redundant undo steps in paint-modes.

For now, don't store undo steps for property changes
in paint & edit modes.

While not ideal, this is similar to 2.7x behavior.

3 files changed, 44 insertions, 0 deletions

diff --git a/source/blender/editors/include/ED_undo.h b/source/blender/editors/include/ED_undo.h
index ebc50e186a2..fa2630ba726 100644
--- a/source/blender/editors/include/ED_undo.h
+++ b/source/blender/editors/include/ED_undo.h
@@ -54,6 +54,9 @@ bool ED_undo_is_valid(const struct bContext *C, const char *undoname);
 
 bool ED_undo_is_memfile_compatible(const struct bContext *C);
 
+/* Unfortunate workaround for limits mixing undo systems. */
+bool ED_undo_is_legacy_compatible_for_property(struct bContext *C, struct ID *id);
+
 void ED_undo_object_editmode_restore_helper(struct bContext *C,
                                             struct Object **object_array,
                                             uint object_array_len,
diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c
index 8ed3f077cbc..0f5f32c46c0 100644
--- a/source/blender/editors/interface/interface_handlers.c
+++ b/source/blender/editors/interface/interface_handlers.c
@@ -756,6 +756,14 @@ static void ui_apply_but_undo(uiBut *but)
       str = "Unknown Action";
     }
 
+    /* Optionally override undo when undo system doesn't support storing properties. */
+    if (but->rnapoin.id.data) {
+      ID *id = but->rnapoin.id.data;
+      if (!ED_undo_is_legacy_compatible_for_property(but->block->evil_C, id)) {
+        str = "";
+      }
+    }
+
     /* delayed, after all other funcs run, popups are closed, etc */
     after = ui_afterfunc_new();
     BLI_strncpy(after->undostr, str, sizeof(after->undostr));
diff --git a/source/blender/editors/undo/ed_undo.c b/source/blender/editors/undo/ed_undo.c
index 6d5f486ebe9..582a2ccaef7 100644
--- a/source/blender/editors/undo/ed_undo.c
+++ b/source/blender/editors/undo/ed_undo.c
@@ -322,6 +322,39 @@ bool ED_undo_is_memfile_compatible(const bContext *C)
 }
 
 /**
+ * When a property of ID changes, return false.
+ *
+ * This is to avoid changes to a property making undo pushes
+ * which are ignored by the undo-system.
+ * For example, changing a brush property isn't stored by sculpt-mode undo steps.
+ * This workaround is needed until the limitation is removed, see: T61948.
+ */
+bool ED_undo_is_legacy_compatible_for_property(struct bContext *C, ID *id)
+{
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  if (view_layer != NULL) {
+    Object *obact = OBACT(view_layer);
+    if (obact != NULL) {
+      if (obact->mode & OB_MODE_ALL_PAINT) {
+        /* Don't store property changes when painting
+         * (only do undo pushes on brush strokes which each paint operator handles on it's own). */
+        CLOG_INFO(&LOG, 1, "skipping undo for paint-mode");
+        return false;
+      }
+      else if (obact->mode & OB_MODE_EDIT) {
+        if ((id == NULL) || (obact->data == NULL) ||
+            (GS(id->name) != GS(((ID *)obact->data)->name))) {
+          /* No undo push on id type mismatch in edit-mode. */
+          CLOG_INFO(&LOG, 1, "skipping undo for edit-mode");
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+/**
  * Ideally we wont access the stack directly,
  * this is needed for modes which handle undo themselves (bypassing #ED_undo_push).
  *