follow up on r59628, setting modes now ensures that other modes exit first,

this was only done in some cases before and it was possible to enable weightpaint+sculpt at the same time when enabling sculpt by directly running the mode switching operator.

add generic function to ensure a compatible mode before entering the new mode (added to each operators exec function)

6 files changed, 149 insertions, 82 deletions

diff --git a/source/blender/editors/include/ED_object.h b/source/blender/editors/include/ED_object.h
index 90b131e5acc..bcc09cced3b 100644
--- a/source/blender/editors/include/ED_object.h
+++ b/source/blender/editors/include/ED_object.h
@@ -123,6 +123,7 @@ struct Base *ED_object_add_duplicate(struct Main *bmain, struct Scene *scene, st
 
 void ED_object_parent(struct Object *ob, struct Object *parent, int type, const char *substr);
 
+bool ED_object_mode_compat_set(struct bContext *C, struct Object *ob, int mode, struct ReportList *reports);
 void ED_object_toggle_modes(struct bContext *C, int mode);
 
 /* bitflags for enter/exit editmode */
diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index b51cfce88b5..aec094d7fc3 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -79,6 +79,7 @@
 #include "BKE_softbody.h"
 #include "BKE_modifier.h"
 #include "BKE_editmesh.h"
+#include "BKE_report.h"
 
 #include "ED_armature.h"
 #include "ED_curve.h"
@@ -556,11 +557,20 @@ void ED_object_editmode_enter(bContext *C, int flag)
 	if (flag & EM_WAITCURSOR) waitcursor(0);
 }
 
-static int editmode_toggle_exec(bContext *C, wmOperator *UNUSED(op))
+static int editmode_toggle_exec(bContext *C, wmOperator *op)
 {
+	const int mode_flag = OB_MODE_EDIT;
+	const bool is_mode_set = CTX_data_edit_object(C);
 	ToolSettings *toolsettings =  CTX_data_tool_settings(C);
 
-	if (!CTX_data_edit_object(C))
+	if (!is_mode_set) {
+		Scene *scene = CTX_data_scene(C);
+		if (!ED_object_mode_compat_set(C, scene->basact->object, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
+	}
+
+	if (!is_mode_set)
 		ED_object_editmode_enter(C, EM_WAITCURSOR);
 	else
 		ED_object_editmode_exit(C, EM_FREEDATA | EM_FREEUNDO | EM_WAITCURSOR);  /* had EM_DO_UNDO but op flag calls undo too [#24685] */
@@ -582,10 +592,7 @@ static int editmode_toggle_poll(bContext *C)
 	if ((ob->restrictflag & OB_RESTRICT_VIEW) && !(ob->mode & OB_MODE_EDIT))
 		return 0;
 
-	return (ob->type == OB_MESH || ob->type == OB_ARMATURE ||
-	        ob->type == OB_FONT || ob->type == OB_MBALL ||
-	        ob->type == OB_LATTICE || ob->type == OB_SURF ||
-	        ob->type == OB_CURVE);
+	return (ELEM7(ob->type, OB_MESH, OB_ARMATURE, OB_FONT, OB_MBALL, OB_LATTICE, OB_SURF, OB_CURVE));
 }
 
 void OBJECT_OT_editmode_toggle(wmOperatorType *ot)
@@ -598,7 +605,6 @@ void OBJECT_OT_editmode_toggle(wmOperatorType *ot)
 	
 	/* api callbacks */
 	ot->exec = editmode_toggle_exec;
-	
 	ot->poll = editmode_toggle_poll;
 	
 	/* flags */
@@ -607,16 +613,25 @@ void OBJECT_OT_editmode_toggle(wmOperatorType *ot)
 
 /* *************************** */
 
-static int posemode_exec(bContext *C, wmOperator *UNUSED(op))
+static int posemode_exec(bContext *C, wmOperator *op)
 {
 	Base *base = CTX_data_active_base(C);
+	Object *ob = base->object;
+	const int mode_flag = OB_MODE_POSE;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
 	
-	if (base->object->type == OB_ARMATURE) {
-		if (base->object == CTX_data_edit_object(C)) {
+	if (!is_mode_set) {
+		if (ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
+	}
+
+	if (ob->type == OB_ARMATURE) {
+		if (ob == CTX_data_edit_object(C)) {
 			ED_object_editmode_exit(C, EM_FREEDATA | EM_DO_UNDO);
 			ED_armature_enter_posemode(C, base);
 		}
-		else if (base->object->mode & OB_MODE_POSE)
+		else if (is_mode_set)
 			ED_armature_exit_posemode(C, base);
 		else
 			ED_armature_enter_posemode(C, base);
@@ -1503,7 +1518,7 @@ static const char *object_mode_op_string(int mode)
 /* checks the mode to be set is compatible with the object
  * should be made into a generic function
  */
-static bool object_mode_set_compat(Object *ob, ObjectMode mode)
+static bool object_mode_compat_test(Object *ob, ObjectMode mode)
 {
 	if (ob) {
 		if (mode == OB_MODE_OBJECT)
@@ -1538,6 +1553,30 @@ static bool object_mode_set_compat(Object *ob, ObjectMode mode)
 	return false;
 }
 
+/**
+ * Sets the mode to a compatible state (use before entering the mode).
+ *
+ * This is so each mode's exec function can call
+ */
+bool ED_object_mode_compat_set(bContext *C, Object *ob, int mode, ReportList *reports)
+{
+	bool ok;
+	if (!ELEM(ob->mode, mode, OB_MODE_OBJECT)) {
+		const char *opstring = object_mode_op_string(ob->mode);
+		WM_operator_name_call(C, opstring, WM_OP_EXEC_REGION_WIN, NULL);
+		ok = ELEM(ob->mode, mode, OB_MODE_OBJECT);
+		if (!ok) {
+			wmOperatorType *ot = WM_operatortype_find(opstring, false);
+			BKE_reportf(reports, RPT_ERROR, "Unable to execute '%s', error changing modes", ot->name);
+		}
+	}
+	else {
+		ok = true;
+	}
+
+	return ok;
+}
+
 static int object_mode_set_exec(bContext *C, wmOperator *op)
 {
 	Object *ob = CTX_data_active_object(C);
@@ -1545,14 +1584,15 @@ static int object_mode_set_exec(bContext *C, wmOperator *op)
 	ObjectMode restore_mode = (ob) ? ob->mode : OB_MODE_OBJECT;
 	int toggle = RNA_boolean_get(op->ptr, "toggle");
 
-	if (!ob || !object_mode_set_compat(ob, mode))
+	if (!ob || !object_mode_compat_test(ob, mode))
 		return OPERATOR_PASS_THROUGH;
 
-	/* Exit current mode if it's not the mode we're setting */
-	if (ob->mode != OB_MODE_OBJECT && ob->mode != mode) {
-		WM_operator_name_call(C, object_mode_op_string(ob->mode), WM_OP_EXEC_REGION_WIN, NULL);
+	if (ob->mode != mode) {
+		/* we should be able to remove this call, each operator calls  */
+		ED_object_mode_compat_set(C, ob, mode, op->reports);
 	}
 
+	/* Exit current mode if it's not the mode we're setting */
 	if (mode != OB_MODE_OBJECT && (ob->mode != mode || toggle)) {
 		/* Enter new mode */
 		WM_operator_name_call(C, object_mode_op_string(mode), WM_OP_EXEC_REGION_WIN, NULL);
diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index d6bb394ff79..dc7ec16d7c1 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -69,6 +69,7 @@
 #include "BIF_gl.h"
 #include "BIF_glutil.h"
 
+#include "ED_object.h"
 #include "ED_physics.h"
 #include "ED_mesh.h"
 #include "ED_particle.h"
@@ -4345,23 +4346,36 @@ static void PE_create_particle_edit(Scene *scene, Object *ob, PointCache *cache,
 
 static int particle_edit_toggle_poll(bContext *C)
 {
-	Scene *scene= CTX_data_scene(C);
-	Object *ob= CTX_data_active_object(C);
+	Object *ob = CTX_data_active_object(C);
 
-	if (!scene || !ob || ob->id.lib)
+	if (ob == NULL || ob->type != OB_MESH)
 		return 0;
-	
-	return (ob->particlesystem.first || modifiers_findByType(ob, eModifierType_Cloth) || modifiers_findByType(ob, eModifierType_Softbody));
+	if (!ob->data || ((ID *)ob->data)->lib)
+		return 0;
+	if (CTX_data_edit_object(C))
+		return 0;
+
+	return (ob->particlesystem.first ||
+	        modifiers_findByType(ob, eModifierType_Cloth) ||
+	        modifiers_findByType(ob, eModifierType_Softbody));
 }
 
-static int particle_edit_toggle_exec(bContext *C, wmOperator *UNUSED(op))
+static int particle_edit_toggle_exec(bContext *C, wmOperator *op)
 {
-	Scene *scene= CTX_data_scene(C);
-	Object *ob= CTX_data_active_object(C);
+	Scene *scene = CTX_data_scene(C);
+	Object *ob = CTX_data_active_object(C);
+	const int mode_flag = OB_MODE_PARTICLE_EDIT;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
+
+	if (!is_mode_set) {
+		if (!ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
+	}
 
-	if (!(ob->mode & OB_MODE_PARTICLE_EDIT)) {
+	if (!is_mode_set) {
 		PTCacheEdit *edit;
-		ob->mode |= OB_MODE_PARTICLE_EDIT;
+		ob->mode |= mode_flag;
 		edit= PE_create_current(scene, ob);
 	
 		/* mesh may have changed since last entering editmode.
@@ -4373,7 +4387,7 @@ static int particle_edit_toggle_exec(bContext *C, wmOperator *UNUSED(op))
 		WM_event_add_notifier(C, NC_SCENE|ND_MODE|NS_MODE_PARTICLE, NULL);
 	}
 	else {
-		ob->mode &= ~OB_MODE_PARTICLE_EDIT;
+		ob->mode &= ~mode_flag;
 		toggle_particle_cursor(C, 0);
 		WM_event_add_notifier(C, NC_SCENE|ND_MODE|NS_MODE_OBJECT, NULL);
 	}
diff --git a/source/blender/editors/sculpt_paint/paint_image.c b/source/blender/editors/sculpt_paint/paint_image.c
index fcf00d4ab2c..feff02fa121 100644
--- a/source/blender/editors/sculpt_paint/paint_image.c
+++ b/source/blender/editors/sculpt_paint/paint_image.c
@@ -82,6 +82,7 @@
 #include "UI_view2d.h"
 
 #include "ED_image.h"
+#include "ED_object.h"
 #include "ED_screen.h"
 #include "ED_sculpt.h"
 #include "ED_uvedit.h"
@@ -1006,9 +1007,12 @@ void PAINT_OT_sample_color(wmOperatorType *ot)
 
 static int texture_paint_toggle_poll(bContext *C)
 {
-	if (CTX_data_edit_object(C))
+	Object *ob = CTX_data_active_object(C);
+	if (ob == NULL || ob->type != OB_MESH)
 		return 0;
-	if (CTX_data_active_object(C) == NULL)
+	if (!ob->data || ((ID *)ob->data)->lib)
+		return 0;
+	if (CTX_data_edit_object(C))
 		return 0;
 
 	return 1;
@@ -1018,25 +1022,20 @@ static int texture_paint_toggle_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
 	Object *ob = CTX_data_active_object(C);
-	Mesh *me = NULL;
-	
-	if (ob == NULL)
-		return OPERATOR_CANCELLED;
-	
-	if (BKE_object_obdata_is_libdata(ob)) {
-		BKE_report(op->reports, RPT_ERROR, "Cannot edit external libdata");
-		return OPERATOR_CANCELLED;
+	const int mode_flag = OB_MODE_TEXTURE_PAINT;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
+	Mesh *me;
+
+	if (!is_mode_set) {
+		if (!ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
 	}
 
 	me = BKE_mesh_from_object(ob);
 
-	if (!(ob->mode & OB_MODE_TEXTURE_PAINT) && !me) {
-		BKE_report(op->reports, RPT_ERROR, "Can only enter texture paint mode for mesh objects");
-		return OPERATOR_CANCELLED;
-	}
-
-	if (ob->mode & OB_MODE_TEXTURE_PAINT) {
-		ob->mode &= ~OB_MODE_TEXTURE_PAINT;
+	if (ob->mode & mode_flag) {
+		ob->mode &= ~mode_flag;
 
 		if (U.glreslimit != 0)
 			GPU_free_images();
@@ -1045,7 +1044,7 @@ static int texture_paint_toggle_exec(bContext *C, wmOperator *op)
 		toggle_paint_cursor(C, 0);
 	}
 	else {
-		ob->mode |= OB_MODE_TEXTURE_PAINT;
+		ob->mode |= mode_flag;
 
 		if (me->mtface == NULL)
 			me->mtface = CustomData_add_layer(&me->fdata, CD_MTFACE, CD_DEFAULT,
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index ddb6a8171d9..d4f8e608f0b 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -69,6 +69,7 @@
 #include "GPU_buffers.h"
 
 #include "ED_armature.h"
+#include "ED_object.h"
 #include "ED_mesh.h"
 #include "ED_screen.h"
 #include "ED_view3d.h"
@@ -76,8 +77,6 @@
 #include "paint_intern.h"  /* own include */
 
 
-static int vpaint_mode_toggle_exec(bContext *C, wmOperator *op);
-
 /* check if we can do partial updates and have them draw realtime
  * (without rebuilding the 'derivedFinal') */
 static int vertex_paint_use_fast_update_check(Object *ob)
@@ -2035,15 +2034,22 @@ static void do_weight_paint_vertex(
 static int wpaint_mode_toggle_exec(bContext *C, wmOperator *op)
 {		
 	Object *ob = CTX_data_active_object(C);
+	const int mode_flag = OB_MODE_WEIGHT_PAINT;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
 	Scene *scene = CTX_data_scene(C);
 	VPaint *wp = scene->toolsettings->wpaint;
 	Mesh *me;
-	
+
+	if (!is_mode_set) {
+		if (!ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
+	}
+
 	me = BKE_mesh_from_object(ob);
-	if (ob->id.lib || me == NULL) return OPERATOR_PASS_THROUGH;
-	
-	if (ob->mode & OB_MODE_WEIGHT_PAINT) {
-		ob->mode &= ~OB_MODE_WEIGHT_PAINT;
+
+	if (ob->mode & mode_flag) {
+		ob->mode &= ~mode_flag;
 
 		if (me->editflag & ME_EDIT_PAINT_VERT_SEL) {
 			BKE_mesh_flush_select_from_verts(me);
@@ -2057,11 +2063,7 @@ static int wpaint_mode_toggle_exec(bContext *C, wmOperator *op)
 		mesh_mirrtopo_table(NULL, 'e');
 	}
 	else {
-		ob->mode |= OB_MODE_WEIGHT_PAINT;
-
-		/* Turn off vertex painting */
-		if (ob->mode & OB_MODE_VERTEX_PAINT)
-			vpaint_mode_toggle_exec(C, op);
+		ob->mode |= mode_flag;
 
 		if (wp == NULL)
 			wp = scene->toolsettings->wpaint = new_vpaint(1);
@@ -2091,12 +2093,12 @@ static int wpaint_mode_toggle_exec(bContext *C, wmOperator *op)
 static int paint_poll_test(bContext *C)
 {
 	Object *ob = CTX_data_active_object(C);
-	if (CTX_data_edit_object(C))
-		return 0;
-	if (CTX_data_active_object(C) == NULL)
+	if (ob == NULL || ob->type != OB_MESH)
 		return 0;
 	if (!ob->data || ((ID *)ob->data)->lib)
 		return 0;
+	if (CTX_data_edit_object(C))
+		return 0;
 	return 1;
 }
 
@@ -2660,35 +2662,35 @@ void PAINT_OT_weight_set(wmOperatorType *ot)
 static int vpaint_mode_toggle_exec(bContext *C, wmOperator *op)
 {	
 	Object *ob = CTX_data_active_object(C);
+	const int mode_flag = OB_MODE_VERTEX_PAINT;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
 	Scene *scene = CTX_data_scene(C);
 	VPaint *vp = scene->toolsettings->vpaint;
 	Mesh *me;
-	
-	me = BKE_mesh_from_object(ob);
-	
-	if (me == NULL || BKE_object_obdata_is_libdata(ob)) {
-		ob->mode &= ~OB_MODE_VERTEX_PAINT;
-		return OPERATOR_PASS_THROUGH;
-	}
-	
-	if (me && me->mloopcol == NULL) {
-		make_vertexcol(ob);
+
+	if (!is_mode_set) {
+		if (!ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
 	}
+
+	me = BKE_mesh_from_object(ob);
 	
 	/* toggle: end vpaint */
-	if (ob->mode & OB_MODE_VERTEX_PAINT) {
-		ob->mode &= ~OB_MODE_VERTEX_PAINT;
+	if (is_mode_set) {
+		ob->mode &= ~mode_flag;
 
 		if (me->editflag & ME_EDIT_PAINT_FACE_SEL) {
 			BKE_mesh_flush_select_from_polys(me);
 		}
 	}
 	else {
-		ob->mode |= OB_MODE_VERTEX_PAINT;
-		/* Turn off weight painting */
-		if (ob->mode & OB_MODE_WEIGHT_PAINT)
-			wpaint_mode_toggle_exec(C, op);
-		
+		ob->mode |= mode_flag;
+
+		if (me->mloopcol == NULL) {
+			make_vertexcol(ob);
+		}
+
 		if (vp == NULL)
 			vp = scene->toolsettings->vpaint = new_vpaint(0);
 		
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index e7b79adc6c7..9046adc6cdf 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -78,6 +78,7 @@
 #include "WM_types.h"
 
 #include "ED_sculpt.h"
+#include "ED_object.h"
 #include "ED_screen.h"
 #include "ED_view3d.h"
 #include "ED_util.h" /* for crazyspace correction */
@@ -4916,21 +4917,31 @@ int ED_sculpt_mask_layers_ensure(Object *ob, MultiresModifierData *mmd)
 	return ret;
 }
 
-static int sculpt_mode_toggle_exec(bContext *C, wmOperator *UNUSED(op))
+static int sculpt_mode_toggle_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
 	ToolSettings *ts = CTX_data_tool_settings(C);
 	Object *ob = CTX_data_active_object(C);
-	Mesh *me = ob->data;
+	const int mode_flag = OB_MODE_SCULPT;
+	const bool is_mode_set = (ob->mode & mode_flag) != 0;
+	Mesh *me;
 	MultiresModifierData *mmd = sculpt_multires_active(scene, ob);
 	int flush_recalc = 0;
 
+	if (!is_mode_set) {
+		if (!ED_object_mode_compat_set(C, ob, mode_flag, op->reports)) {
+			return OPERATOR_CANCELLED;
+		}
+	}
+
+	me = BKE_mesh_from_object(ob);
+
 	/* multires in sculpt mode could have different from object mode subdivision level */
 	flush_recalc |= mmd && mmd->sculptlvl != mmd->lvl;
 	/* if object has got active modifiers, it's dm could be different in sculpt mode  */
 	flush_recalc |= sculpt_has_active_modifiers(scene, ob);
 
-	if (ob->mode & OB_MODE_SCULPT) {
+	if (is_mode_set) {
 		if (mmd)
 			multires_force_update(ob);
 
@@ -4945,13 +4956,13 @@ static int sculpt_mode_toggle_exec(bContext *C, wmOperator *UNUSED(op))
 		}
 
 		/* Leave sculptmode */
-		ob->mode &= ~OB_MODE_SCULPT;
+		ob->mode &= ~mode_flag;
 
 		free_sculptsession(ob);
 	}
 	else {
 		/* Enter sculptmode */
-		ob->mode |= OB_MODE_SCULPT;
+		ob->mode |= mode_flag;
 
 		/* Remove dynamic-topology flag; this will be enabled if the
 		 * file was saved with dynamic topology on, but we don't