XR Controller Support Step 5: Navigation

Adds navigation transforms (pose, scale) to the XR session state that
will be applied to the viewer/controller poses. By manipulating these
values, a viewer can move through the VR viewport without the need to
physically walk through it.

Add-ons can access these transforms via Python
(XrSessionState.navigation_location/rotation/scale) to use with custom
operators.

Also adds 3 new VR navigation operators that will be exposed to users
as default actions in the VR Scene Inspection add-on. While all three
of these operators have custom properties that can greatly influence
their behaviors, for now these properties will not be accessible by
users from the UI. However, other add-ons can still set these custom
properties if they desire.

1). Raycast-based teleport
Moves the user to a location pointed at on a mesh object. The result
can optionally be constrained to specific axes, for example to achieve
"elevation snapping" behavior by constraining to the Z-axis. In
addition, one can specify an interpolation factor and offset.

Credit to KISKA for the elevation snapping concept.

2). "Grab" navigation
Moves the user through the viewport by pressing inputs on one or two
held controllers and applying deltas to the navigation matrix based on
the displacement of these controllers. When inputs on both controllers
are pressed at the same time (bimanual interaction), the user can scale
themselves relative to the scene based on the distance between the
controllers.

Also supports locks for location, rotation, and scale.

3). Fly navigation
Navigates the viewport by pressing a button and moving/turning relative to
navigation space or the VR viewer or controller. Via the operator's
properties, one can select from a variety of these modes as well as
specify the min/max speed and whether to lock elevation.

Reviewed By: Severin

Differential Revision: https://developer.blender.org/D11501

1 files changed, 150 insertions, 34 deletions

diff --git a/source/blender/windowmanager/xr/intern/wm_xr_session.c b/source/blender/windowmanager/xr/intern/wm_xr_session.c
index 9f53db1347c..62757c0bddd 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_session.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr_session.c
@@ -66,11 +66,20 @@ static void wm_xr_session_create_cb(void)
   Main *bmain = G_MAIN;
   wmWindowManager *wm = bmain->wm.first;
   wmXrData *xr_data = &wm->xr;
+  wmXrSessionState *state = &xr_data->runtime->session_state;
+  XrSessionSettings *settings = &xr_data->session_settings;
 
   /* Get action set data from Python. */
   BKE_callback_exec_null(bmain, BKE_CB_EVT_XR_SESSION_START_PRE);
 
   wm_xr_session_actions_init(xr_data);
+
+  /* Initialize navigation. */
+  WM_xr_session_state_navigation_reset(state);
+  if (settings->base_scale < FLT_EPSILON) {
+    settings->base_scale = 1.0f;
+  }
+  state->prev_base_scale = settings->base_scale;
 }
 
 static void wm_xr_session_controller_data_free(wmXrSessionState *state)
@@ -167,7 +176,8 @@ bool WM_xr_session_is_ready(const wmXrData *xr)
 
 static void wm_xr_session_base_pose_calc(const Scene *scene,
                                          const XrSessionSettings *settings,
-                                         GHOST_XrPose *r_base_pose)
+                                         GHOST_XrPose *r_base_pose,
+                                         float *r_base_scale)
 {
   const Object *base_pose_object = ((settings->base_pose_type == XR_BASE_POSE_OBJECT) &&
                                     settings->base_pose_object) ?
@@ -198,6 +208,8 @@ static void wm_xr_session_base_pose_calc(const Scene *scene,
     copy_v3_fl(r_base_pose->position, 0.0f);
     axis_angle_to_quat_single(r_base_pose->orientation_quat, 'X', M_PI_2);
   }
+
+  *r_base_scale = settings->base_scale;
 }
 
 static void wm_xr_session_draw_data_populate(wmXrData *xr_data,
@@ -213,7 +225,8 @@ static void wm_xr_session_draw_data_populate(wmXrData *xr_data,
   r_draw_data->xr_data = xr_data;
   r_draw_data->surface_data = g_xr_surface->customdata;
 
-  wm_xr_session_base_pose_calc(r_draw_data->scene, settings, &r_draw_data->base_pose);
+  wm_xr_session_base_pose_calc(
+      r_draw_data->scene, settings, &r_draw_data->base_pose, &r_draw_data->base_scale);
 }
 
 wmWindow *wm_xr_session_root_window_or_fallback_get(const wmWindowManager *wm,
@@ -291,7 +304,7 @@ static wmXrSessionStateEvent wm_xr_session_state_to_event(const wmXrSessionState
   return SESSION_STATE_EVENT_NONE;
 }
 
-void wm_xr_session_draw_data_update(const wmXrSessionState *state,
+void wm_xr_session_draw_data_update(wmXrSessionState *state,
                                     const XrSessionSettings *settings,
                                     const GHOST_XrDrawViewInfo *draw_view,
                                     wmXrDrawData *draw_data)
@@ -319,6 +332,8 @@ void wm_xr_session_draw_data_update(const wmXrSessionState *state,
       else {
         copy_v3_fl(draw_data->eye_position_ofs, 0.0f);
       }
+      /* Reset navigation. */
+      WM_xr_session_state_navigation_reset(state);
       break;
     case SESSION_STATE_EVENT_POSITION_TRACKING_TOGGLE:
       if (use_position_tracking) {
@@ -345,32 +360,36 @@ void wm_xr_session_draw_data_update(const wmXrSessionState *state,
 void wm_xr_session_state_update(const XrSessionSettings *settings,
                                 const wmXrDrawData *draw_data,
                                 const GHOST_XrDrawViewInfo *draw_view,
+                                const float viewmat[4][4],
                                 wmXrSessionState *state)
 {
   GHOST_XrPose viewer_pose;
-  const bool use_position_tracking = settings->flag & XR_SESSION_USE_POSITION_TRACKING;
-  const bool use_absolute_tracking = settings->flag & XR_SESSION_USE_ABSOLUTE_TRACKING;
-
-  mul_qt_qtqt(viewer_pose.orientation_quat,
-              draw_data->base_pose.orientation_quat,
-              draw_view->local_pose.orientation_quat);
-  copy_v3_v3(viewer_pose.position, draw_data->base_pose.position);
-  /* The local pose and the eye pose (which is copied from an earlier local pose) both are view
-   * space, so Y-up. In this case we need them in regular Z-up. */
-  if (use_position_tracking) {
-    viewer_pose.position[0] += draw_view->local_pose.position[0];
-    viewer_pose.position[1] -= draw_view->local_pose.position[2];
-    viewer_pose.position[2] += draw_view->local_pose.position[1];
-  }
-  if (!use_absolute_tracking) {
-    viewer_pose.position[0] -= draw_data->eye_position_ofs[0];
-    viewer_pose.position[1] += draw_data->eye_position_ofs[2];
-    viewer_pose.position[2] -= draw_data->eye_position_ofs[1];
-  }
-
-  copy_v3_v3(state->viewer_pose.position, viewer_pose.position);
-  copy_qt_qt(state->viewer_pose.orientation_quat, viewer_pose.orientation_quat);
-  wm_xr_pose_to_imat(&viewer_pose, state->viewer_viewmat);
+  float viewer_mat[4][4], base_mat[4][4], nav_mat[4][4];
+
+  /* Calculate viewer matrix. */
+  copy_qt_qt(viewer_pose.orientation_quat, draw_view->local_pose.orientation_quat);
+  if ((settings->flag & XR_SESSION_USE_POSITION_TRACKING) == 0) {
+    zero_v3(viewer_pose.position);
+  }
+  else {
+    copy_v3_v3(viewer_pose.position, draw_view->local_pose.position);
+  }
+  if ((settings->flag & XR_SESSION_USE_ABSOLUTE_TRACKING) == 0) {
+    sub_v3_v3(viewer_pose.position, draw_data->eye_position_ofs);
+  }
+  wm_xr_pose_to_mat(&viewer_pose, viewer_mat);
+
+  /* Apply base pose and navigation. */
+  wm_xr_pose_scale_to_mat(&draw_data->base_pose, draw_data->base_scale, base_mat);
+  wm_xr_pose_scale_to_mat(&state->nav_pose_prev, state->nav_scale_prev, nav_mat);
+  mul_m4_m4m4(state->viewer_mat_base, base_mat, viewer_mat);
+  mul_m4_m4m4(viewer_mat, nav_mat, state->viewer_mat_base);
+
+  /* Save final viewer pose and viewmat. */
+  mat4_to_loc_quat(state->viewer_pose.position, state->viewer_pose.orientation_quat, viewer_mat);
+  wm_xr_pose_scale_to_imat(
+      &state->viewer_pose, draw_data->base_scale * state->nav_scale_prev, state->viewer_viewmat);
+
   /* No idea why, but multiplying by two seems to make it match the VR view more. */
   state->focal_len = 2.0f *
                      fov_to_focallength(draw_view->fov.angle_right - draw_view->fov.angle_left,
@@ -378,7 +397,10 @@ void wm_xr_session_state_update(const XrSessionSettings *settings,
 
   copy_v3_v3(state->prev_eye_position_ofs, draw_data->eye_position_ofs);
   memcpy(&state->prev_base_pose, &draw_data->base_pose, sizeof(state->prev_base_pose));
+  state->prev_base_scale = draw_data->base_scale;
   memcpy(&state->prev_local_pose, &draw_view->local_pose, sizeof(state->prev_local_pose));
+  copy_v3_v3(state->prev_eye_position_ofs, draw_data->eye_position_ofs);
+
   state->prev_settings_flag = settings->flag;
   state->prev_base_pose_type = settings->base_pose_type;
   state->prev_base_pose_object = settings->base_pose_object;
@@ -503,6 +525,74 @@ bool WM_xr_session_state_controller_aim_rotation_get(const wmXrData *xr,
   return true;
 }
 
+bool WM_xr_session_state_nav_location_get(const wmXrData *xr, float r_location[3])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
+    zero_v3(r_location);
+    return false;
+  }
+
+  copy_v3_v3(r_location, xr->runtime->session_state.nav_pose.position);
+  return true;
+}
+
+void WM_xr_session_state_nav_location_set(wmXrData *xr, const float location[3])
+{
+  if (WM_xr_session_exists(xr)) {
+    copy_v3_v3(xr->runtime->session_state.nav_pose.position, location);
+    xr->runtime->session_state.is_navigation_dirty = true;
+  }
+}
+
+bool WM_xr_session_state_nav_rotation_get(const wmXrData *xr, float r_rotation[4])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
+    unit_qt(r_rotation);
+    return false;
+  }
+
+  copy_qt_qt(r_rotation, xr->runtime->session_state.nav_pose.orientation_quat);
+  return true;
+}
+
+void WM_xr_session_state_nav_rotation_set(wmXrData *xr, const float rotation[4])
+{
+  if (WM_xr_session_exists(xr)) {
+    BLI_ASSERT_UNIT_QUAT(rotation);
+    copy_qt_qt(xr->runtime->session_state.nav_pose.orientation_quat, rotation);
+    xr->runtime->session_state.is_navigation_dirty = true;
+  }
+}
+
+bool WM_xr_session_state_nav_scale_get(const wmXrData *xr, float *r_scale)
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set) {
+    *r_scale = 1.0f;
+    return false;
+  }
+
+  *r_scale = xr->runtime->session_state.nav_scale;
+  return true;
+}
+
+void WM_xr_session_state_nav_scale_set(wmXrData *xr, float scale)
+{
+  if (WM_xr_session_exists(xr)) {
+    /* Clamp to reasonable values. */
+    CLAMP(scale, xr->session_settings.clip_start, xr->session_settings.clip_end);
+    xr->runtime->session_state.nav_scale = scale;
+    xr->runtime->session_state.is_navigation_dirty = true;
+  }
+}
+
+void WM_xr_session_state_navigation_reset(wmXrSessionState *state)
+{
+  zero_v3(state->nav_pose.position);
+  unit_qt(state->nav_pose.orientation_quat);
+  state->nav_scale = 1.0f;
+  state->is_navigation_dirty = true;
+}
+
 /* -------------------------------------------------------------------- */
 /** \name XR-Session Actions
  *
@@ -522,16 +612,21 @@ void wm_xr_session_actions_init(wmXrData *xr)
 static void wm_xr_session_controller_pose_calc(const GHOST_XrPose *raw_pose,
                                                const float view_ofs[3],
                                                const float base_mat[4][4],
+                                               const float nav_mat[4][4],
                                                GHOST_XrPose *r_pose,
-                                               float r_mat[4][4])
+                                               float r_mat[4][4],
+                                               float r_mat_base[4][4])
 {
   float m[4][4];
   /* Calculate controller matrix in world space. */
   wm_xr_pose_to_mat(raw_pose, m);
 
-  /* Apply eye position and base pose offsets. */
+  /* Apply eye position offset. */
   sub_v3_v3(m[3], view_ofs);
-  mul_m4_m4m4(r_mat, base_mat, m);
+
+  /* Apply base pose and navigation. */
+  mul_m4_m4m4(r_mat_base, base_mat, m);
+  mul_m4_m4m4(r_mat, nav_mat, r_mat_base);
 
   /* Save final pose. */
   mat4_to_loc_quat(r_pose->position, r_pose->orientation_quat, r_mat);
@@ -547,7 +642,7 @@ static void wm_xr_session_controller_data_update(const XrSessionSettings *settin
   BLI_assert(grip_action->count_subaction_paths == BLI_listbase_count(&state->controllers));
 
   unsigned int subaction_idx = 0;
-  float view_ofs[3], base_mat[4][4];
+  float view_ofs[3], base_mat[4][4], nav_mat[4][4];
 
   if ((settings->flag & XR_SESSION_USE_POSITION_TRACKING) == 0) {
     copy_v3_v3(view_ofs, state->prev_local_pose.position);
@@ -559,19 +654,24 @@ static void wm_xr_session_controller_data_update(const XrSessionSettings *settin
     add_v3_v3(view_ofs, state->prev_eye_position_ofs);
   }
 
-  wm_xr_pose_to_mat(&state->prev_base_pose, base_mat);
+  wm_xr_pose_scale_to_mat(&state->prev_base_pose, state->prev_base_scale, base_mat);
+  wm_xr_pose_scale_to_mat(&state->nav_pose, state->nav_scale, nav_mat);
 
   LISTBASE_FOREACH_INDEX (wmXrController *, controller, &state->controllers, subaction_idx) {
     wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)grip_action->states)[subaction_idx],
                                        view_ofs,
                                        base_mat,
+                                       nav_mat,
                                        &controller->grip_pose,
-                                       controller->grip_mat);
+                                       controller->grip_mat,
+                                       controller->grip_mat_base);
     wm_xr_session_controller_pose_calc(&((GHOST_XrPose *)aim_action->states)[subaction_idx],
                                        view_ofs,
                                        base_mat,
+                                       nav_mat,
                                        &controller->aim_pose,
-                                       controller->aim_mat);
+                                       controller->aim_mat,
+                                       controller->aim_mat_base);
 
     if (!controller->model) {
       /* Notify GHOST to load/continue loading the controller model data. This can be called more
@@ -1094,10 +1194,26 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
     return;
   }
 
+  XrSessionSettings *settings = &xr->session_settings;
   GHOST_XrContextHandle xr_context = xr->runtime->context;
   wmXrSessionState *state = &xr->runtime->session_state;
   wmXrActionSet *active_action_set = state->active_action_set;
 
+  if (state->is_navigation_dirty) {
+    memcpy(&state->nav_pose_prev, &state->nav_pose, sizeof(state->nav_pose_prev));
+    state->nav_scale_prev = state->nav_scale;
+    state->is_navigation_dirty = false;
+
+    /* Update viewer pose with any navigation changes since the last actions sync so that data
+     * is correct for queries. */
+    float m[4][4], viewer_mat[4][4];
+    wm_xr_pose_scale_to_mat(&state->nav_pose, state->nav_scale, m);
+    mul_m4_m4m4(viewer_mat, m, state->viewer_mat_base);
+    mat4_to_loc_quat(state->viewer_pose.position, state->viewer_pose.orientation_quat, viewer_mat);
+    wm_xr_pose_scale_to_imat(
+        &state->viewer_pose, settings->base_scale * state->nav_scale, state->viewer_viewmat);
+  }
+
   int ret = GHOST_XrSyncActions(xr_context, active_action_set ? active_action_set->name : NULL);
   if (!ret) {
     return;
@@ -1108,7 +1224,7 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
     wmWindow *win = wm_xr_session_root_window_or_fallback_get(wm, xr->runtime);
 
     if (active_action_set->controller_grip_action && active_action_set->controller_aim_action) {
-      wm_xr_session_controller_data_update(&xr->session_settings,
+      wm_xr_session_controller_data_update(settings,
                                            active_action_set->controller_grip_action,
                                            active_action_set->controller_aim_action,
                                            xr_context,