Fix [#23977] toggle back to object mode not working (outliner issue)

Reported by Roland Kramer

There was already code to prevent visibility toggle through restrict column from working when in edit mode. Reshuffled
code somewhat so it works also for object operations in outliner. Also ensure operator poll for visibility and selectability toggle
checks object is not in edit mode. So this also works for selectability toggling, so no more toggling when in edit mode - it's confusing otherwise.

Added notifier and handling for it for renderability toggle in outliner. No edit mode restriction here.

7 files changed, 75 insertions, 33 deletions

diff --git a/source/blender/editors/include/ED_screen.h b/source/blender/editors/include/ED_screen.h
index 203d93150eb..d95011c5961 100644
--- a/source/blender/editors/include/ED_screen.h
+++ b/source/blender/editors/include/ED_screen.h
@@ -128,6 +128,7 @@ int		ED_operator_view3d_active(struct bContext *C);
 int		ED_operator_region_view3d_active(struct bContext *C);
 int		ED_operator_timeline_active(struct bContext *C);
 int		ED_operator_outliner_active(struct bContext *C);
+int		ED_operator_outliner_active_no_editobject(struct bContext *C);
 int		ED_operator_file_active(struct bContext *C);
 int		ED_operator_action_active(struct bContext *C);
 int		ED_operator_buttons_active(struct bContext *C);
diff --git a/source/blender/editors/object/object_select.c b/source/blender/editors/object/object_select.c
index fa9a97f4e74..818bb50a1fb 100644
--- a/source/blender/editors/object/object_select.c
+++ b/source/blender/editors/object/object_select.c
@@ -72,7 +72,8 @@
  * this takes into account the 'restrict selection in 3d view' flag.
  * deselect works always, the restriction just prevents selection */
 
-/* Note: send a NC_SCENE|ND_OB_SELECT notifier yourself! */
+/* Note: send a NC_SCENE|ND_OB_SELECT notifier yourself! (or 
+ * or a NC_SCENE|ND_OB_VISIBLE in case of visibility toggling */
 
 void ED_base_object_select(Base *base, short mode)
 {
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index a77e1990c04..7336d762225 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -148,6 +148,19 @@ int ED_operator_outliner_active(bContext *C)
 	return ed_spacetype_test(C, SPACE_OUTLINER);
 }
 
+int ED_operator_outliner_active_no_editobject(bContext *C)
+{
+	if(ed_spacetype_test(C, SPACE_OUTLINER)) {
+		Object *ob = ED_object_active_context(C);
+		Object *obedit= CTX_data_edit_object(C);
+		if(ob && ob == obedit)
+			return 0;
+		else
+			return 1;
+	}
+	return 0;
+}
+
 int ED_operator_file_active(bContext *C)
 {
 	return ed_spacetype_test(C, SPACE_FILE);
diff --git a/source/blender/editors/space_outliner/outliner.c b/source/blender/editors/space_outliner/outliner.c
index 175eb8c78ed..aaeab9e7843 100644
--- a/source/blender/editors/space_outliner/outliner.c
+++ b/source/blender/editors/space_outliner/outliner.c
@@ -1553,10 +1553,37 @@ static void outliner_set_flag(SpaceOops *soops, ListBase *lb, short flag, short
 
 /* --- */
 
+/* same check needed for both object operation and restrict column button func
+ * return 0 when in edit mode (cannot restrict view or select)
+ * otherwise return 1 */
+static int common_restrict_check(bContext *C, Scene *scene, Object *ob)
+{
+	/* Don't allow hide an object in edit mode,
+	 * check the bug #22153 and #21609, #23977
+	 */
+	Object *obedit= CTX_data_edit_object(C);
+	if (obedit && obedit == ob) {
+		/* found object is hidden, reset */
+		if (ob->restrictflag & OB_RESTRICT_VIEW)
+			ob->restrictflag &= ~OB_RESTRICT_VIEW;
+		/* found object is unselectable, reset */
+		if (ob->restrictflag & OB_RESTRICT_SELECT)
+			ob->restrictflag &= ~OB_RESTRICT_SELECT;
+		return 0;
+	}
+	
+	return 1;
+}
+
 void object_toggle_visibility_cb(bContext *C, Scene *scene, TreeElement *te, TreeStoreElem *tsep, TreeStoreElem *tselem)
 {
 	Base *base= (Base *)te->directdata;
-	if(base || (base= object_in_scene((Object *)tselem->id, scene))) {
+	Object *ob = (Object *)tselem->id;
+	
+	/* add check for edit mode */
+	if(!common_restrict_check(C, scene, ob)) return;
+	
+	if(base || (base= object_in_scene(ob, scene))) {
 		if((base->object->restrictflag ^= OB_RESTRICT_VIEW)) {
 			ED_base_object_select(base, BA_DESELECT);
 		}
@@ -1586,7 +1613,7 @@ void OUTLINER_OT_visibility_toggle(wmOperatorType *ot)
 	
 	/* callbacks */
 	ot->exec= outliner_toggle_visibility_exec;
-	ot->poll= ED_operator_outliner_active;
+	ot->poll= ED_operator_outliner_active_no_editobject;
 	
 	ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
 }
@@ -1626,7 +1653,7 @@ void OUTLINER_OT_selectability_toggle(wmOperatorType *ot)
 	
 	/* callbacks */
 	ot->exec= outliner_toggle_selectability_exec;
-	ot->poll= ED_operator_outliner_active;
+	ot->poll= ED_operator_outliner_active_no_editobject;
 	
 	ot->flag= OPTYPE_REGISTER|OPTYPE_UNDO;
 }
@@ -3403,6 +3430,7 @@ static int outliner_object_operation_exec(bContext *C, wmOperator *op)
 	else if(event==8) {
 		outliner_do_object_operation(C, scene, soops, &soops->tree, object_toggle_renderability_cb);
 		str= "Toggle Renderability";
+		WM_event_add_notifier(C, NC_SCENE|ND_OB_RENDER, scene);
 	}
 
 	ED_undo_push(C, str);
@@ -4843,16 +4871,8 @@ static void restrictbutton_view_cb(bContext *C, void *poin, void *poin2)
 {
 	Scene *scene = (Scene *)poin;
 	Object *ob = (Object *)poin2;
-	Object *obedit= CTX_data_edit_object(C);
 
-	/* Don't allow hide an objet in edit mode,
-	 * check the bug #22153 and #21609
-	 */
-	if (obedit && obedit == ob) {
-		if (ob->restrictflag & OB_RESTRICT_VIEW)
-			ob->restrictflag &= ~OB_RESTRICT_VIEW;
-		return;
-	}
+	if(!common_restrict_check(C, scene, ob)) return;
 	
 	/* deselect objects that are invisible */
 	if (ob->restrictflag & OB_RESTRICT_VIEW) {
@@ -4869,6 +4889,8 @@ static void restrictbutton_sel_cb(bContext *C, void *poin, void *poin2)
 	Scene *scene = (Scene *)poin;
 	Object *ob = (Object *)poin2;
 	
+	if(!common_restrict_check(C, scene, ob)) return;
+	
 	/* if select restriction has just been turned on */
 	if (ob->restrictflag & OB_RESTRICT_SELECT) {
 		/* Ouch! There is no backwards pointer from Object to Base, 
@@ -4881,7 +4903,7 @@ static void restrictbutton_sel_cb(bContext *C, void *poin, void *poin2)
 
 static void restrictbutton_rend_cb(bContext *C, void *poin, void *poin2)
 {
-	WM_event_add_notifier(C, NC_SCENE|ND_OB_SELECT, poin);
+	WM_event_add_notifier(C, NC_SCENE|ND_OB_RENDER, poin);
 }
 
 static void restrictbutton_r_lay_cb(bContext *C, void *poin, void *poin2)
diff --git a/source/blender/editors/space_outliner/space_outliner.c b/source/blender/editors/space_outliner/space_outliner.c
index 5e11d3502c1..d91c8caa14c 100644
--- a/source/blender/editors/space_outliner/space_outliner.c
+++ b/source/blender/editors/space_outliner/space_outliner.c
@@ -97,6 +97,8 @@ static void outliner_main_area_listener(ARegion *ar, wmNotifier *wmn)
 			switch(wmn->data) {
 				case ND_OB_ACTIVE:
 				case ND_OB_SELECT:
+				case ND_OB_VISIBLE:
+				case ND_OB_RENDER:
 				case ND_MODE:
 				case ND_KEYINGSET:
 				case ND_FRAME:
diff --git a/source/blender/editors/space_view3d/space_view3d.c b/source/blender/editors/space_view3d/space_view3d.c
index c6b92fae1c0..a49ada8a4c9 100644
--- a/source/blender/editors/space_view3d/space_view3d.c
+++ b/source/blender/editors/space_view3d/space_view3d.c
@@ -726,6 +726,7 @@ static void view3d_header_area_listener(ARegion *ar, wmNotifier *wmn)
 				case ND_FRAME:
 				case ND_OB_ACTIVE:
 				case ND_OB_SELECT:
+				case ND_OB_VISIBLE:
 				case ND_MODE:
 				case ND_LAYER:
 				case ND_TOOLSETTINGS:
@@ -779,6 +780,7 @@ static void view3d_buttons_area_listener(ARegion *ar, wmNotifier *wmn)
 				case ND_FRAME:
 				case ND_OB_ACTIVE:
 				case ND_OB_SELECT:
+				case ND_OB_VISIBLE:
 				case ND_MODE:
 				case ND_LAYER:
 				case ND_LAYER_CONTENT:
diff --git a/source/blender/windowmanager/WM_types.h b/source/blender/windowmanager/WM_types.h
index 3f67888a4e9..9b18a2cd9ba 100644
--- a/source/blender/windowmanager/WM_types.h
+++ b/source/blender/windowmanager/WM_types.h
@@ -178,28 +178,29 @@ typedef struct wmNotifier {
 #define ND_OB_ACTIVE		(7<<16)
 #define ND_OB_SELECT		(8<<16)
 #define ND_OB_VISIBLE		(9<<16)
-#define ND_MODE				(10<<16)
-#define ND_RENDER_RESULT	(11<<16)
-#define ND_COMPO_RESULT		(12<<16)
-#define ND_KEYINGSET		(13<<16)
-#define ND_TOOLSETTINGS		(14<<16)
-#define ND_LAYER			(15<<16)
-#define ND_FRAME_RANGE		(16<<16)
+#define ND_OB_RENDER		(10<<16)
+#define ND_MODE				(11<<16)
+#define ND_RENDER_RESULT	(12<<16)
+#define ND_COMPO_RESULT		(13<<16)
+#define ND_KEYINGSET		(14<<16)
+#define ND_TOOLSETTINGS		(15<<16)
+#define ND_LAYER			(16<<16)
+#define ND_FRAME_RANGE		(17<<16)
 #define ND_LAYER_CONTENT	(101<<16)
 
 	/* NC_OBJECT Object */
-#define	ND_TRANSFORM		(17<<16)
-#define ND_OB_SHADING		(18<<16)
-#define ND_POSE				(19<<16)
-#define ND_BONE_ACTIVE		(20<<16)
-#define ND_BONE_SELECT		(21<<16)
-#define ND_DRAW				(22<<16)
-#define ND_MODIFIER			(23<<16)
-#define ND_KEYS				(24<<16)
-#define ND_CONSTRAINT		(25<<16)
-#define ND_PARTICLE			(26<<16)
-#define ND_POINTCACHE		(27<<16)
-#define ND_PARENT			(28<<16)
+#define	ND_TRANSFORM		(18<<16)
+#define ND_OB_SHADING		(19<<16)
+#define ND_POSE				(20<<16)
+#define ND_BONE_ACTIVE		(21<<16)
+#define ND_BONE_SELECT		(22<<16)
+#define ND_DRAW				(23<<16)
+#define ND_MODIFIER			(24<<16)
+#define ND_KEYS				(25<<16)
+#define ND_CONSTRAINT		(26<<16)
+#define ND_PARTICLE			(27<<16)
+#define ND_POINTCACHE		(28<<16)
+#define ND_PARENT			(29<<16)
 
 	/* NC_MATERIAL Material */
 #define	ND_SHADING			(30<<16)