sanity checks on operator exec/modal/invoke return values.

3 files changed, 24 insertions, 9 deletions

diff --git a/source/blender/makesdna/DNA_windowmanager_types.h b/source/blender/makesdna/DNA_windowmanager_types.h
index 1f0ae28a00d..47ebf111eba 100644
--- a/source/blender/makesdna/DNA_windowmanager_types.h
+++ b/source/blender/makesdna/DNA_windowmanager_types.h
@@ -320,14 +320,20 @@ typedef struct wmOperator {
 
 } wmOperator;
 
-/* operator type exec(), invoke() modal(), return values */
-#define OPERATOR_RUNNING_MODAL	1
-#define OPERATOR_CANCELLED		2
-#define OPERATOR_FINISHED		4
+
+/* operator type return flags: exec(), invoke() modal(), return values */
+#define OPERATOR_RUNNING_MODAL	(1<<0)
+#define OPERATOR_CANCELLED		(1<<1)
+#define OPERATOR_FINISHED		(1<<2)
 /* add this flag if the event should pass through */
-#define OPERATOR_PASS_THROUGH	8
+#define OPERATOR_PASS_THROUGH	(1<<3)
 /* in case operator got executed outside WM code... like via fileselect */
-#define OPERATOR_HANDLED		16
+#define OPERATOR_HANDLED		(1<<4)
+
+#define OPERATOR_FLAGS_ALL		((1<<5)-1)
+
+/* sanity checks for debug mode only */
+#define OPERATOR_RETVAL_CHECK(ret) BLI_assert(ret != 0 && (ret & OPERATOR_FLAGS_ALL) == ret)
 
 /* wmOperator flag */
 #define OP_GRAB_POINTER			1
diff --git a/source/blender/windowmanager/intern/wm_event_system.c b/source/blender/windowmanager/intern/wm_event_system.c
index 413ff181f11..27586525253 100644
--- a/source/blender/windowmanager/intern/wm_event_system.c
+++ b/source/blender/windowmanager/intern/wm_event_system.c
@@ -531,6 +531,7 @@ static int wm_operator_exec(bContext *C, wmOperator *op, int repeat)
 			wm->op_undo_depth++;
 
 		retval= op->type->exec(C, op);
+		OPERATOR_RETVAL_CHECK(retval);
 
 		if(op->type->flag & OPTYPE_UNDO && CTX_wm_manager(C) == wm)
 			wm->op_undo_depth--;
@@ -690,6 +691,7 @@ static int wm_operator_invoke(bContext *C, wmOperatorType *ot, wmEvent *event, P
 				wm->op_undo_depth++;
 
 			retval= op->type->invoke(C, op, event);
+			OPERATOR_RETVAL_CHECK(retval);
 
 			if(op->type->flag & OPTYPE_UNDO && CTX_wm_manager(C) == wm)
 				wm->op_undo_depth--;
@@ -699,6 +701,7 @@ static int wm_operator_invoke(bContext *C, wmOperatorType *ot, wmEvent *event, P
 				wm->op_undo_depth++;
 
 			retval= op->type->exec(C, op);
+			OPERATOR_RETVAL_CHECK(retval);
 
 			if(op->type->flag & OPTYPE_UNDO && CTX_wm_manager(C) == wm)
 				wm->op_undo_depth--;
@@ -917,6 +920,7 @@ int WM_operator_call_py(bContext *C, wmOperatorType *ot, int context, PointerRNA
 			wm->op_undo_depth++;
 
 		retval= op->type->exec(C, op);
+		OPERATOR_RETVAL_CHECK(retval);
 
 		if(op->type->flag & OPTYPE_UNDO && CTX_wm_manager(C) == wm)
 			wm->op_undo_depth--;
@@ -1203,6 +1207,7 @@ static int wm_handler_operator_call(bContext *C, ListBase *handlers, wmEventHand
 				wm->op_undo_depth++;
 
 			retval= ot->modal(C, op, event);
+			OPERATOR_RETVAL_CHECK(retval);
 
 			if(ot->flag & OPTYPE_UNDO && CTX_wm_manager(C) == wm)
 				wm->op_undo_depth--;
diff --git a/source/blender/windowmanager/intern/wm_operators.c b/source/blender/windowmanager/intern/wm_operators.c
index d794685ea70..a4efa8fff84 100644
--- a/source/blender/windowmanager/intern/wm_operators.c
+++ b/source/blender/windowmanager/intern/wm_operators.c
@@ -213,6 +213,7 @@ static int wm_macro_exec(bContext *C, wmOperator *op)
 		
 		if(opm->type->exec) {
 			retval= opm->type->exec(C, opm);
+			OPERATOR_RETVAL_CHECK(retval);
 		
 			if (retval & OPERATOR_FINISHED) {
 				MacroData *md = op->customdata;
@@ -237,6 +238,8 @@ static int wm_macro_invoke_internal(bContext *C, wmOperator *op, wmEvent *event,
 		else if(opm->type->exec)
 			retval= opm->type->exec(C, opm);
 
+		OPERATOR_RETVAL_CHECK(retval);
+
 		BLI_movelisttolist(&op->reports->list, &opm->reports->list);
 		
 		if (retval & OPERATOR_FINISHED) {
@@ -265,6 +268,7 @@ static int wm_macro_modal(bContext *C, wmOperator *op, wmEvent *event)
 		printf("macro error, calling NULL modal()\n");
 	else {
 		retval = opm->type->modal(C, opm, event);
+		OPERATOR_RETVAL_CHECK(retval);
 
 		/* if this one is done but it's not the last operator in the macro */
 		if ((retval & OPERATOR_FINISHED) && opm->next) {
@@ -655,7 +659,9 @@ int WM_menu_invoke(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
 		printf("WM_menu_invoke: %s \"%s\" is not an enum property\n", op->type->idname, RNA_property_identifier(prop));
 	}
 	else if (RNA_property_is_set(op->ptr, RNA_property_identifier(prop))) {
-		return op->type->exec(C, op);
+		const int retval= op->type->exec(C, op);
+		OPERATOR_RETVAL_CHECK(retval);
+		return retval;
 	}
 	else {
 		pup= uiPupMenuBegin(C, op->type->name, ICON_NONE);
@@ -2345,7 +2351,6 @@ static void gesture_circle_apply(bContext *C, wmOperator *op)
 	
 	if(op->type->exec)
 		op->type->exec(C, op);
-
 #ifdef GESTURE_MEMORY
 	circle_select_size= rect->xmax;
 #endif
@@ -2566,7 +2571,6 @@ static void gesture_lasso_apply(bContext *C, wmOperator *op)
 		
 	if(op->type->exec)
 		op->type->exec(C, op);
-	
 }
 
 int WM_gesture_lasso_modal(bContext *C, wmOperator *op, wmEvent *event)