Object: refactor mode switching

Functionality here has become confusing over time,
this removes duplicate, similar functions, preferring to set the mode
instead of toggle, enter, exit.

Mode switching utility function behaved differently regarding undo,
'ED_object_mode_toggle' for example didn't skip the undo push
where 'ED_object_mode_set' did.
Some callers chose these functions based on the intended undo behavior,
even when toggling didn't make sense.

There was also ED_object_mode_generic_enter which was similar to
ED_object_mode_set, instead of the reverse of ED_object_mode_generic_exit.

Simplify object mode switching internals:

- Replace ED_object_mode_generic_enter with ED_object_mode_set.
- Remove ED_object_mode_toggle as nearly all callers needed to check
  the current mode so toggling would set the mode argument correctly.
- Use ED_object_mode_set for the object mode switching operator
  to simplify logic.
- Add ED_object_mode_set_ex which has an argument to disable undo,
  needed when loading undo data needs to set the mode.
- Remove unused ED_object_mode_exit.

2 files changed, 43 insertions, 67 deletions

diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index 8fce6dce26a..d522dcabae3 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -1495,10 +1495,6 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
    * Notes:
    * - Code below avoids calling mode switching functions more than once,
    *   as this causes unnecessary calculations and undo steps to be added.
-   * - Even though toggle is called (#ED_object_mode_toggle),
-   *   this is just used to enable/disable the mode based on checking the current mode.
-   * - Code would read better if we used #ED_object_mode_set,
-   *   this needs some refactoring as it handles undo differently (TODO).
    * - The previous mode (#Object.restore_mode) is object mode by default.
    *
    * Supported Cases:
@@ -1513,14 +1509,7 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
    */
   if (toggle == false) {
     if (ob->mode != mode) {
-      if (mode == OB_MODE_OBJECT) {
-        /* Exit the current mode into object mode. */
-        ED_object_mode_compat_set(C, ob, OB_MODE_OBJECT, op->reports);
-      }
-      else {
-        /* Enter 'mode'. */
-        ED_object_mode_toggle(C, mode);
-      }
+      ED_object_mode_set_ex(C, mode, true, op->reports);
     }
   }
   else {
@@ -1529,16 +1518,14 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
      * otherwise there is nothing to do. */
     if (mode == OB_MODE_OBJECT) {
       if (ob->mode != OB_MODE_OBJECT) {
-        /* Set object mode if the object is not already in object mode. */
-        if (ED_object_mode_compat_set(C, ob, OB_MODE_OBJECT, op->reports)) {
+        if (ED_object_mode_set_ex(C, OB_MODE_OBJECT, true, op->reports)) {
           /* Store old mode so we know what to go back to. */
           ob->restore_mode = mode_prev;
         }
       }
       else {
         if (ob->restore_mode != OB_MODE_OBJECT) {
-          /* Enter 'restore_mode'. */
-          ED_object_mode_toggle(C, ob->restore_mode);
+          ED_object_mode_set_ex(C, ob->restore_mode, true, op->reports);
         }
       }
     }
@@ -1546,20 +1533,17 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
       /* Non-object modes, enter the 'mode' unless it's already set,
        * in that case use restore mode. */
       if (ob->mode != mode) {
-        ED_object_mode_toggle(C, mode);
-        if (ob->mode == mode) {
+        if (ED_object_mode_set_ex(C, mode, true, op->reports)) {
           /* Store old mode so we know what to go back to. */
           ob->restore_mode = mode_prev;
         }
       }
       else {
         if (ob->restore_mode != OB_MODE_OBJECT) {
-          /* Enter 'restore_mode'. */
-          ED_object_mode_toggle(C, ob->restore_mode);
+          ED_object_mode_set_ex(C, ob->restore_mode, true, op->reports);
         }
         else {
-          /* Enter 'mode'. */
-          ED_object_mode_toggle(C, mode);
+          ED_object_mode_set_ex(C, OB_MODE_OBJECT, true, op->reports);
         }
       }
     }
diff --git a/source/blender/editors/object/object_modes.c b/source/blender/editors/object/object_modes.c
index f06b6a4db2a..fbd935c2e55 100644
--- a/source/blender/editors/object/object_modes.c
+++ b/source/blender/editors/object/object_modes.c
@@ -167,41 +167,6 @@ bool ED_object_mode_compat_set(bContext *C, Object *ob, eObjectMode mode, Report
   return ok;
 }
 
-void ED_object_mode_toggle(bContext *C, eObjectMode mode)
-{
-  if (mode != OB_MODE_OBJECT) {
-    const char *opstring = object_mode_op_string(mode);
-
-    if (opstring) {
-      wmOperatorType *ot = WM_operatortype_find(opstring, false);
-      WM_operator_name_call_ptr(C, ot, WM_OP_EXEC_REGION_WIN, NULL);
-    }
-  }
-}
-
-/* Wrapper for operator  */
-void ED_object_mode_set(bContext *C, eObjectMode mode)
-{
-  wmWindowManager *wm = CTX_wm_manager(C);
-  wm->op_undo_depth++;
-  /* needed so we don't do undo pushes. */
-  ED_object_mode_generic_enter(C, mode);
-  wm->op_undo_depth--;
-}
-
-void ED_object_mode_exit(bContext *C, Depsgraph *depsgraph)
-{
-  struct Main *bmain = CTX_data_main(C);
-  Scene *scene = CTX_data_scene(C);
-  ViewLayer *view_layer = CTX_data_view_layer(C);
-  FOREACH_OBJECT_BEGIN (view_layer, ob) {
-    if (ob->mode & OB_MODE_ALL_MODE_DATA) {
-      ED_object_mode_generic_exit(bmain, depsgraph, scene, ob);
-    }
-  }
-  FOREACH_OBJECT_END;
-}
-
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -212,23 +177,50 @@ void ED_object_mode_exit(bContext *C, Depsgraph *depsgraph)
  *
  * \{ */
 
-bool ED_object_mode_generic_enter(struct bContext *C, eObjectMode object_mode)
+bool ED_object_mode_set_ex(bContext *C, eObjectMode mode, bool use_undo, ReportList *reports)
 {
+  wmWindowManager *wm = CTX_wm_manager(C);
   ViewLayer *view_layer = CTX_data_view_layer(C);
   Object *ob = OBACT(view_layer);
   if (ob == NULL) {
-    return (object_mode == OB_MODE_OBJECT);
+    return (mode == OB_MODE_OBJECT);
+  }
+
+  if ((ob->type == OB_GPENCIL) && (mode == OB_MODE_EDIT)) {
+    mode = OB_MODE_EDIT_GPENCIL;
   }
-  if (ob->mode == object_mode) {
+
+  if (ob->mode == mode) {
     return true;
   }
-  wmOperatorType *ot = WM_operatortype_find("OBJECT_OT_mode_set", false);
-  PointerRNA ptr;
-  WM_operator_properties_create_ptr(&ptr, ot);
-  RNA_enum_set(&ptr, "mode", object_mode);
-  WM_operator_name_call_ptr(C, ot, WM_OP_INVOKE_DEFAULT, &ptr);
-  WM_operator_properties_free(&ptr);
-  return (ob->mode == object_mode);
+
+  if (!ED_object_mode_compat_test(ob, mode)) {
+    return false;
+  }
+
+  const char *opstring = object_mode_op_string((mode == OB_MODE_OBJECT) ? ob->mode : mode);
+  wmOperatorType *ot = WM_operatortype_find(opstring, false);
+
+  if (!use_undo) {
+    wm->op_undo_depth++;
+  }
+  WM_operator_name_call_ptr(C, ot, WM_OP_EXEC_REGION_WIN, NULL);
+  if (!use_undo) {
+    wm->op_undo_depth--;
+  }
+
+  if (ob->mode != mode) {
+    BKE_reportf(reports, RPT_ERROR, "Unable to execute '%s', error changing modes", ot->name);
+    return false;
+  }
+
+  return true;
+}
+
+bool ED_object_mode_set(bContext *C, eObjectMode mode)
+{
+  /* Don't do undo push by default, since this may be called by lower level code. */
+  return ED_object_mode_set_ex(C, mode, true, NULL);
 }
 
 /**