Manipulator: modal callback can now cancel & pass events

Re-use operator return flags for manipulator modal & invoke,
this means manipulators can allow navigation or other events to be
handled as they run - see T52499

5 files changed, 47 insertions, 36 deletions

diff --git a/source/blender/windowmanager/manipulators/intern/wm_manipulator.c b/source/blender/windowmanager/manipulators/intern/wm_manipulator.c
index fe7dbb56d6e..316254202fa 100644
--- a/source/blender/windowmanager/manipulators/intern/wm_manipulator.c
+++ b/source/blender/windowmanager/manipulators/intern/wm_manipulator.c
@@ -204,7 +204,7 @@ void WM_manipulator_unlink(ListBase *manipulatorlist, wmManipulatorMap *mmap, wm
 		wm_manipulatormap_highlight_set(mmap, C, NULL, 0);
 	}
 	if (mpr->state & WM_MANIPULATOR_STATE_MODAL) {
-		wm_manipulatormap_modal_set(mmap, C, NULL, NULL);
+		wm_manipulatormap_modal_set(mmap, C, mpr, NULL, false);
 	}
 	/* Unlink instead of setting so we don't run callbacks. */
 	if (mpr->state & WM_MANIPULATOR_STATE_SELECT) {
diff --git a/source/blender/windowmanager/manipulators/intern/wm_manipulator_group.c b/source/blender/windowmanager/manipulators/intern/wm_manipulator_group.c
index 6f322933e2a..808051560bc 100644
--- a/source/blender/windowmanager/manipulators/intern/wm_manipulator_group.c
+++ b/source/blender/windowmanager/manipulators/intern/wm_manipulator_group.c
@@ -98,7 +98,7 @@ void wm_manipulatorgroup_free(bContext *C, wmManipulatorGroup *mgroup)
 		wm_manipulatormap_highlight_set(mmap, C, NULL, 0);
 	}
 	if (mmap->mmap_context.modal && mmap->mmap_context.modal->parent_mgroup == mgroup) {
-		wm_manipulatormap_modal_set(mmap, C, NULL, NULL);
+		wm_manipulatormap_modal_set(mmap, C, mmap->mmap_context.modal, NULL, false);
 	}
 
 	for (wmManipulator *mpr = mgroup->manipulators.first, *mpr_next; mpr; mpr = mpr_next) {
@@ -313,7 +313,7 @@ static void manipulator_tweak_finish(bContext *C, wmOperator *op, const bool can
 	if (mtweak->mpr_modal->type->exit) {
 		mtweak->mpr_modal->type->exit(C, mtweak->mpr_modal, cancel);
 	}
-	wm_manipulatormap_modal_set(mtweak->mmap, C, NULL, NULL);
+	wm_manipulatormap_modal_set(mtweak->mmap, C, mtweak->mpr_modal, NULL, false);
 	MEM_freeN(mtweak);
 }
 
@@ -359,11 +359,12 @@ static int manipulator_tweak_modal(bContext *C, wmOperator *op, const wmEvent *e
 	}
 
 	/* handle manipulator */
-	if (mpr->custom_modal) {
-		mpr->custom_modal(C, mpr, event, mtweak->flag);
-	}
-	else if (mpr->type->modal) {
-		mpr->type->modal(C, mpr, event, mtweak->flag);
+	wmManipulatorFnModal modal_fn = mpr->custom_modal ? mpr->custom_modal : mpr->type->modal;
+	int retval = modal_fn(C, mpr, event, mtweak->flag);
+
+	if ((retval & OPERATOR_RUNNING_MODAL) == 0) {
+		manipulator_tweak_finish(C, op, (retval & OPERATOR_CANCELLED) != 0);
+		return OPERATOR_FINISHED;
 	}
 
 	/* always return PASS_THROUGH so modal handlers
@@ -385,7 +386,7 @@ static int manipulator_tweak_invoke(bContext *C, wmOperator *op, const wmEvent *
 
 
 	/* activate highlighted manipulator */
-	wm_manipulatormap_modal_set(mmap, C, event, mpr);
+	wm_manipulatormap_modal_set(mmap, C, mpr, event, true);
 
 	/* XXX temporary workaround for modal manipulator operator
 	 * conflicting with modal operator attached to manipulator */
@@ -395,6 +396,10 @@ static int manipulator_tweak_invoke(bContext *C, wmOperator *op, const wmEvent *
 		}
 	}
 
+	/* Couldn't start the manipulator. */
+	if ((mpr->state & WM_MANIPULATOR_STATE_MODAL) == 0) {
+		return OPERATOR_PASS_THROUGH;
+	}
 
 	ManipulatorTweakData *mtweak = MEM_mallocN(sizeof(ManipulatorTweakData), __func__);
 
diff --git a/source/blender/windowmanager/manipulators/intern/wm_manipulator_map.c b/source/blender/windowmanager/manipulators/intern/wm_manipulator_map.c
index 4273bc75cc9..9155c03db2a 100644
--- a/source/blender/windowmanager/manipulators/intern/wm_manipulator_map.c
+++ b/source/blender/windowmanager/manipulators/intern/wm_manipulator_map.c
@@ -638,18 +638,21 @@ void wm_manipulatormaps_handled_modal_update(
 		if (mpr && (mpr->op_data.type != NULL) &&
 		    (mpr->op_data.type == handler->op->type))
 		{
-			if (mpr->custom_modal) {
-				mpr->custom_modal(C, mpr, event, 0);
-			}
-			else if (mpr->type->modal) {
-				mpr->type->modal(C, mpr, event, 0);
+			wmManipulatorFnModal modal_fn = mpr->custom_modal ? mpr->custom_modal : mpr->type->modal;
+			if (modal_fn != NULL) {
+				int retval = modal_fn(C, mpr, event, 0);
+				/* The manipulator is tried to the operator, we can't choose when to exit. */
+				BLI_assert(retval & OPERATOR_RUNNING_MODAL);
+				UNUSED_VARS_NDEBUG(retval);
 			}
 		}
 	}
 	/* operator not running anymore */
 	else {
 		wm_manipulatormap_highlight_set(mmap, C, NULL, 0);
-		wm_manipulatormap_modal_set(mmap, C, event, NULL);
+		if (mpr) {
+			wm_manipulatormap_modal_set(mmap, C, mpr, NULL, false);
+		}
 	}
 
 	/* restore the area */
@@ -836,24 +839,33 @@ wmManipulator *wm_manipulatormap_highlight_get(wmManipulatorMap *mmap)
 	return mmap->mmap_context.highlight;
 }
 
+/**
+ * Caller should call exit when (enable == False).
+ */
 void wm_manipulatormap_modal_set(
-        wmManipulatorMap *mmap, bContext *C, const wmEvent *event, wmManipulator *mpr)
+        wmManipulatorMap *mmap, bContext *C, wmManipulator *mpr, const wmEvent *event, bool enable)
 {
-	if (mpr && C) {
+	if (enable) {
+		BLI_assert(mmap->mmap_context.modal == NULL);
+
 		/* For now only grab cursor for 3D manipulators. */
 		bool grab_cursor = (mpr->parent_mgroup->type->flag & WM_MANIPULATORGROUPTYPE_3D) != 0;
+		int retval = OPERATOR_RUNNING_MODAL;
+
+		if (mpr->type->invoke &&
+		    (mpr->type->modal || mpr->custom_modal))
+		{
+			retval = mpr->type->invoke(C, mpr, event);
+		}
+
+		if ((retval & OPERATOR_RUNNING_MODAL) == 0) {
+			return;
+		}
 
 		mpr->state |= WM_MANIPULATOR_STATE_MODAL;
 		mmap->mmap_context.modal = mpr;
 
 		if (mpr->op_data.type) {
-			/* first activate the manipulator itself */
-			if (mpr->type->invoke &&
-			    (mpr->type->modal || mpr->custom_modal))
-			{
-				mpr->type->invoke(C, mpr, event);
-			}
-
 			WM_operator_name_call_ptr(C, mpr->op_data.type, WM_OP_INVOKE_DEFAULT, &mpr->op_data.ptr);
 
 			/* we failed to hook the manipulator to the operator handler or operator was cancelled, return */
@@ -863,20 +875,13 @@ void wm_manipulatormap_modal_set(
 			}
 			return;
 		}
-		else {
-			if (mpr->type->invoke &&
-			    (mpr->type->modal || mpr->custom_modal))
-			{
-				mpr->type->invoke(C, mpr, event);
-			}
-		}
 
 		if (grab_cursor) {
 			WM_cursor_grab_enable(CTX_wm_window(C), true, true, NULL);
 		}
 	}
 	else {
-		mpr = mmap->mmap_context.modal;
+		BLI_assert(ELEM(mmap->mmap_context.modal, NULL, mpr));
 
 		/* deactivate, manipulator but first take care of some stuff */
 		if (mpr) {
diff --git a/source/blender/windowmanager/manipulators/wm_manipulator_fn.h b/source/blender/windowmanager/manipulators/wm_manipulator_fn.h
index a92648455d4..d44d010df68 100644
--- a/source/blender/windowmanager/manipulators/wm_manipulator_fn.h
+++ b/source/blender/windowmanager/manipulators/wm_manipulator_fn.h
@@ -50,10 +50,10 @@ typedef void    (*wmManipulatorFnSetup)(struct wmManipulator *);
 typedef void    (*wmManipulatorFnDraw)(const struct bContext *, struct wmManipulator *);
 typedef void    (*wmManipulatorFnDrawSelect)(const struct bContext *, struct wmManipulator *, int);
 typedef int     (*wmManipulatorFnTestSelect)(struct bContext *, struct wmManipulator *, const struct wmEvent *);
-typedef void    (*wmManipulatorFnModal)(struct bContext *, struct wmManipulator *, const struct wmEvent *, eWM_ManipulatorTweak);
+typedef int     (*wmManipulatorFnModal)(struct bContext *, struct wmManipulator *, const struct wmEvent *, eWM_ManipulatorTweak);
 typedef void    (*wmManipulatorFnPropertyUpdate)(struct wmManipulator *, struct wmManipulatorProperty *);
 typedef void    (*wmManipulatorFnMatrixWorldGet)(const struct wmManipulator *, float[4][4]);
-typedef void    (*wmManipulatorFnInvoke)(struct bContext *, struct wmManipulator *, const struct wmEvent *);
+typedef int     (*wmManipulatorFnInvoke)(struct bContext *, struct wmManipulator *, const struct wmEvent *);
 typedef void    (*wmManipulatorFnExit)(struct bContext *, struct wmManipulator *, const bool);
 typedef int     (*wmManipulatorFnCursorGet)(struct wmManipulator *);
 typedef void    (*wmManipulatorFnSelectRefresh)(struct wmManipulator *);
diff --git a/source/blender/windowmanager/manipulators/wm_manipulator_wmapi.h b/source/blender/windowmanager/manipulators/wm_manipulator_wmapi.h
index cc1bf398764..87cf711a60b 100644
--- a/source/blender/windowmanager/manipulators/wm_manipulator_wmapi.h
+++ b/source/blender/windowmanager/manipulators/wm_manipulator_wmapi.h
@@ -82,8 +82,9 @@ void wm_manipulatormap_highlight_set(
         struct wmManipulator *mpr, int part);
 struct wmManipulator *wm_manipulatormap_highlight_get(struct wmManipulatorMap *mmap);
 void wm_manipulatormap_modal_set(
-        struct wmManipulatorMap *mmap, bContext *C,
-        const struct wmEvent *event, struct wmManipulator *mpr);
+        struct wmManipulatorMap *mmap, bContext *C, struct wmManipulator *mpr,
+        const struct wmEvent *event, bool enable);
+
 struct wmManipulator *wm_manipulatormap_modal_get(struct wmManipulatorMap *mmap);
 struct wmManipulator **wm_manipulatormap_selected_get(wmManipulatorMap *mmap, int *r_selected_len);
 struct ListBase *wm_manipulatormap_groups_get(wmManipulatorMap *mmap);