Object: Mode switching operator

Remove unused OBJECT_OT_mode_set_or_submode, add
OBJECT_OT_mode_set_with_submode which can switch to edit mode as well
as a sub-mode - currently only mesh select mode is supported
(others may be added later).

6 files changed, 90 insertions, 35 deletions

diff --git a/source/blender/editors/include/ED_mesh.h b/source/blender/editors/include/ED_mesh.h
index ee4798430ef..d8d62ad6f08 100644
--- a/source/blender/editors/include/ED_mesh.h
+++ b/source/blender/editors/include/ED_mesh.h
@@ -214,11 +214,12 @@ void EDBM_selectmode_convert(struct BMEditMesh *em,
                              const short selectmode_new);
 
 /* user access this */
-bool EDBM_selectmode_toggle(struct bContext *C,
-                            const short selectmode_new,
-                            const int action,
-                            const bool use_extend,
-                            const bool use_expand);
+bool EDBM_selectmode_set_multi(struct bContext *C, const short selectmode);
+bool EDBM_selectmode_toggle_multi(struct bContext *C,
+                                  const short selectmode_new,
+                                  const int action,
+                                  const bool use_extend,
+                                  const bool use_expand);
 
 bool EDBM_selectmode_disable(struct Scene *scene,
                              struct BMEditMesh *em,
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index ce8d26542d9..d11a223f455 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -1343,7 +1343,7 @@ static int edbm_select_mode_exec(bContext *C, wmOperator *op)
   const bool use_extend = RNA_boolean_get(op->ptr, "use_extend");
   const bool use_expand = RNA_boolean_get(op->ptr, "use_expand");
 
-  if (EDBM_selectmode_toggle(C, type, action, use_extend, use_expand)) {
+  if (EDBM_selectmode_toggle_multi(C, type, action, use_extend, use_expand)) {
     return OPERATOR_FINISHED;
   }
   else {
@@ -2346,11 +2346,11 @@ void EDBM_selectmode_convert(BMEditMesh *em,
 }
 
 /* user facing function, does notification */
-bool EDBM_selectmode_toggle(bContext *C,
-                            const short selectmode_new,
-                            const int action,
-                            const bool use_extend,
-                            const bool use_expand)
+bool EDBM_selectmode_toggle_multi(bContext *C,
+                                  const short selectmode_new,
+                                  const int action,
+                                  const bool use_extend,
+                                  const bool use_expand)
 {
   Scene *scene = CTX_data_scene(C);
   ViewLayer *view_layer = CTX_data_view_layer(C);
@@ -2472,6 +2472,54 @@ bool EDBM_selectmode_toggle(bContext *C,
   return ret;
 }
 
+bool EDBM_selectmode_set_multi(bContext *C, const short selectmode)
+{
+  BLI_assert(selectmode != 0);
+  bool changed = false;
+
+  {
+    Object *obedit = CTX_data_edit_object(C);
+    BMEditMesh *em = NULL;
+    if (obedit && obedit->type == OB_MESH) {
+      em = BKE_editmesh_from_object(obedit);
+    }
+    if (em == NULL) {
+      return changed;
+    }
+  }
+
+  ViewLayer *view_layer = CTX_data_view_layer(C);
+  Scene *scene = CTX_data_scene(C);
+  ToolSettings *ts = scene->toolsettings;
+
+  if (ts->selectmode != selectmode) {
+    ts->selectmode = selectmode;
+    changed = true;
+  }
+
+  uint objects_len = 0;
+  Object **objects = BKE_view_layer_array_from_objects_in_edit_mode_unique_data(
+      view_layer, CTX_wm_view3d(C), &objects_len);
+
+  for (uint ob_index = 0; ob_index < objects_len; ob_index++) {
+    Object *ob_iter = objects[ob_index];
+    BMEditMesh *em_iter = BKE_editmesh_from_object(ob_iter);
+    if (em_iter->selectmode != ts->selectmode) {
+      em_iter->selectmode = ts->selectmode;
+      EDBM_selectmode_set(em_iter);
+      DEG_id_tag_update(ob_iter->data, ID_RECALC_COPY_ON_WRITE | ID_RECALC_SELECT);
+      WM_event_add_notifier(C, NC_GEOM | ND_SELECT, ob_iter->data);
+      changed = true;
+    }
+  }
+
+  if (changed) {
+    WM_main_add_notifier(NC_SCENE | ND_TOOLSETTINGS, NULL);
+    DEG_id_tag_update(&scene->id, ID_RECALC_COPY_ON_WRITE);
+  }
+  return changed;
+}
+
 /**
  * Use to disable a selectmode if its enabled, Using another mode as a fallback
  * if the disabled mode is the only mode set.
diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index ecd8d27bf1c..37063a42906 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -1342,27 +1342,12 @@ static bool object_mode_set_poll(bContext *C)
 
 static int object_mode_set_exec(bContext *C, wmOperator *op)
 {
-  bool use_submode = STREQ(op->idname, "OBJECT_OT_mode_set_or_submode");
+  bool use_submode = STREQ(op->idname, "OBJECT_OT_mode_set_with_submode");
   Object *ob = CTX_data_active_object(C);
   eObjectMode mode = RNA_enum_get(op->ptr, "mode");
   eObjectMode restore_mode = (ob) ? ob->mode : OB_MODE_OBJECT;
   const bool toggle = RNA_boolean_get(op->ptr, "toggle");
 
-  if (use_submode) {
-    /* When not changing modes use submodes, see: T55162. */
-    if (toggle == false) {
-      if (mode == restore_mode) {
-        switch (mode) {
-          case OB_MODE_EDIT:
-            WM_menu_name_call(C, "VIEW3D_MT_edit_mesh_select_mode", WM_OP_INVOKE_REGION_WIN);
-            return OPERATOR_INTERFACE;
-          default:
-            break;
-        }
-      }
-    }
-  }
-
   /* by default the operator assume is a mesh, but if gp object change mode */
   if ((ob != NULL) && (ob->type == OB_GPENCIL) && (mode == OB_MODE_EDIT)) {
     mode = OB_MODE_EDIT_GPENCIL;
@@ -1406,6 +1391,20 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
     }
   }
 
+  if (use_submode) {
+    if (ob->type == OB_MESH) {
+      if (ob->mode & OB_MODE_EDIT) {
+        PropertyRNA *prop = RNA_struct_find_property(op->ptr, "mesh_select_mode");
+        if (RNA_property_is_set(op->ptr, prop)) {
+          int mesh_select_mode = RNA_property_enum_get(op->ptr, prop);
+          if (mesh_select_mode != 0) {
+            EDBM_selectmode_set_multi(C, mesh_select_mode);
+          }
+        }
+      }
+    }
+  }
+
   return OPERATOR_FINISHED;
 }
 
@@ -1435,13 +1434,20 @@ void OBJECT_OT_mode_set(wmOperatorType *ot)
   RNA_def_property_flag(prop, PROP_SKIP_SAVE);
 }
 
-void OBJECT_OT_mode_set_or_submode(wmOperatorType *ot)
+void OBJECT_OT_mode_set_with_submode(wmOperatorType *ot)
 {
   OBJECT_OT_mode_set(ot);
 
   /* identifiers */
-  ot->name = "Set Object Mode or Submode";
-  ot->idname = "OBJECT_OT_mode_set_or_submode";
+  ot->name = "Set Object Mode with Submode";
+  ot->idname = "OBJECT_OT_mode_set_with_submode";
+
+  /* properties */
+  /* we could add other types - particle for eg. */
+  PropertyRNA *prop;
+  prop = RNA_def_enum_flag(
+      ot->srna, "mesh_select_mode", rna_enum_mesh_select_mode_items, 0, "Mesh Mode", "");
+  RNA_def_property_flag(prop, PROP_HIDDEN | PROP_SKIP_SAVE);
 }
 
 static ListBase selected_objects_get(bContext *C)
diff --git a/source/blender/editors/object/object_intern.h b/source/blender/editors/object/object_intern.h
index 4b369c10e4d..669f3c44644 100644
--- a/source/blender/editors/object/object_intern.h
+++ b/source/blender/editors/object/object_intern.h
@@ -71,7 +71,7 @@ void OBJECT_OT_hide_view_set(struct wmOperatorType *ot);
 void OBJECT_OT_hide_view_clear(struct wmOperatorType *ot);
 void OBJECT_OT_hide_collection(struct wmOperatorType *ot);
 void OBJECT_OT_mode_set(struct wmOperatorType *ot);
-void OBJECT_OT_mode_set_or_submode(struct wmOperatorType *ot);
+void OBJECT_OT_mode_set_with_submode(struct wmOperatorType *ot);
 void OBJECT_OT_editmode_toggle(struct wmOperatorType *ot);
 void OBJECT_OT_posemode_toggle(struct wmOperatorType *ot);
 void OBJECT_OT_proxy_make(struct wmOperatorType *ot);
diff --git a/source/blender/editors/object/object_ops.c b/source/blender/editors/object/object_ops.c
index 38c06319450..a1937c40191 100644
--- a/source/blender/editors/object/object_ops.c
+++ b/source/blender/editors/object/object_ops.c
@@ -59,7 +59,7 @@ void ED_operatortypes_object(void)
   WM_operatortype_append(OBJECT_OT_origin_set);
 
   WM_operatortype_append(OBJECT_OT_mode_set);
-  WM_operatortype_append(OBJECT_OT_mode_set_or_submode);
+  WM_operatortype_append(OBJECT_OT_mode_set_with_submode);
   WM_operatortype_append(OBJECT_OT_editmode_toggle);
   WM_operatortype_append(OBJECT_OT_posemode_toggle);
   WM_operatortype_append(OBJECT_OT_proxy_make);
diff --git a/source/blender/editors/space_view3d/view3d_header.c b/source/blender/editors/space_view3d/view3d_header.c
index 6984562337c..200cc12a6c1 100644
--- a/source/blender/editors/space_view3d/view3d_header.c
+++ b/source/blender/editors/space_view3d/view3d_header.c
@@ -109,17 +109,17 @@ static void do_view3d_header_buttons(bContext *C, void *UNUSED(arg), int event)
 
   switch (event) {
     case B_SEL_VERT:
-      if (EDBM_selectmode_toggle(C, SCE_SELECT_VERTEX, -1, shift, ctrl)) {
+      if (EDBM_selectmode_toggle_multi(C, SCE_SELECT_VERTEX, -1, shift, ctrl)) {
         ED_undo_push(C, "Selectmode Set: Vertex");
       }
       break;
     case B_SEL_EDGE:
-      if (EDBM_selectmode_toggle(C, SCE_SELECT_EDGE, -1, shift, ctrl)) {
+      if (EDBM_selectmode_toggle_multi(C, SCE_SELECT_EDGE, -1, shift, ctrl)) {
         ED_undo_push(C, "Selectmode Set: Edge");
       }
       break;
     case B_SEL_FACE:
-      if (EDBM_selectmode_toggle(C, SCE_SELECT_FACE, -1, shift, ctrl)) {
+      if (EDBM_selectmode_toggle_multi(C, SCE_SELECT_FACE, -1, shift, ctrl)) {
         ED_undo_push(C, "Selectmode Set: Face");
       }
       break;