XR Controller Support Step 1: Internal Abstractions for OpenXR Actions

Adds internal API for creating and managing OpenXR actions at the
GHOST and WM layers. Does not bring about any changes for users since
XR action functionality is not yet exposed in the Python API (will be
added in a subsequent patch).

OpenXR actions are a means to communicate with XR input devices and
can be used to retrieve button/pose states or apply haptic feedback.
Actions are bound to device inputs via a semantic path binding
(https://www.khronos.org/registry/OpenXR/specs/1.0/html/xrspec.html#semantic-path-interaction-profiles),
which serves as an XR version of keymaps.

Main features:

- Abstraction of OpenXR action management functions to GHOST-XR,
  WM-XR APIs.
- New "xr_session_start_pre" callback for creating actions at
  appropriate point in the XR session.
- Creation of name-identifiable action sets/actions.
- Binding of actions to controller inputs.
- Acquisition of controller button states.
- Acquisition of controller poses.
- Application of controller haptic feedback.
- Carefully designed error handling and useful error reporting
  (e.g. action set/action name included in error message).

Reviewed By: Julian Eisel

Differential Revision: http://developer.blender.org/D10942

5 files changed, 707 insertions, 0 deletions

diff --git a/source/blender/windowmanager/xr/intern/wm_xr.c b/source/blender/windowmanager/xr/intern/wm_xr.c
index 439d611b085..2a67c2bee9f 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr.c
@@ -128,6 +128,11 @@ bool wm_xr_events_handle(wmWindowManager *wm)
   if (wm->xr.runtime && wm->xr.runtime->context) {
     GHOST_XrEventsHandle(wm->xr.runtime->context);
 
+    /* Process OpenXR action events. */
+    if (WM_xr_session_is_ready(&wm->xr)) {
+      wm_xr_session_actions_update(&wm->xr);
+    }
+
     /* wm_window_process_events() uses the return value to determine if it can put the main thread
      * to sleep for some milliseconds. We never want that to happen while the VR session runs on
      * the main thread. So always return true. */
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_actions.c b/source/blender/windowmanager/xr/intern/wm_xr_actions.c
new file mode 100644
index 00000000000..51ed3dcfd3c
--- /dev/null
+++ b/source/blender/windowmanager/xr/intern/wm_xr_actions.c
@@ -0,0 +1,480 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup wm
+ *
+ * \name Window-Manager XR Actions
+ *
+ * Uses the Ghost-XR API to manage OpenXR actions.
+ * All functions are designed to be usable by RNA / the Python API.
+ */
+
+#include "BLI_math.h"
+
+#include "GHOST_C-api.h"
+
+#include "MEM_guardedalloc.h"
+
+#include "WM_api.h"
+#include "WM_types.h"
+
+#include "wm_xr_intern.h"
+
+/* -------------------------------------------------------------------- */
+/** \name XR-Action API
+ *
+ * API functions for managing OpenXR actions.
+ *
+ * \{ */
+
+static wmXrActionSet *action_set_create(const char *action_set_name)
+{
+  wmXrActionSet *action_set = MEM_callocN(sizeof(*action_set), __func__);
+  action_set->name = MEM_mallocN(strlen(action_set_name) + 1, "XrActionSet_Name");
+  strcpy(action_set->name, action_set_name);
+
+  return action_set;
+}
+
+static void action_set_destroy(void *val)
+{
+  wmXrActionSet *action_set = val;
+
+  MEM_SAFE_FREE(action_set->name);
+
+  MEM_freeN(action_set);
+}
+
+static wmXrActionSet *action_set_find(wmXrData *xr, const char *action_set_name)
+{
+  return GHOST_XrGetActionSetCustomdata(xr->runtime->context, action_set_name);
+}
+
+static wmXrAction *action_create(const char *action_name,
+                                 eXrActionType type,
+                                 unsigned int count_subaction_paths,
+                                 const char **subaction_paths,
+                                 const float *float_threshold,
+                                 wmOperatorType *ot,
+                                 IDProperty *op_properties,
+                                 eXrOpFlag op_flag)
+{
+  wmXrAction *action = MEM_callocN(sizeof(*action), __func__);
+  action->name = MEM_mallocN(strlen(action_name) + 1, "XrAction_Name");
+  strcpy(action->name, action_name);
+  action->type = type;
+
+  const unsigned int count = count_subaction_paths;
+  action->count_subaction_paths = count;
+
+  action->subaction_paths = MEM_mallocN(sizeof(*action->subaction_paths) * count,
+                                        "XrAction_SubactionPaths");
+  for (unsigned int i = 0; i < count; ++i) {
+    action->subaction_paths[i] = MEM_mallocN(strlen(subaction_paths[i]) + 1,
+                                             "XrAction_SubactionPath");
+    strcpy(action->subaction_paths[i], subaction_paths[i]);
+  }
+
+  size_t size;
+  switch (type) {
+    case XR_BOOLEAN_INPUT:
+      size = sizeof(bool);
+      break;
+    case XR_FLOAT_INPUT:
+      size = sizeof(float);
+      break;
+    case XR_VECTOR2F_INPUT:
+      size = sizeof(float) * 2;
+      break;
+    case XR_POSE_INPUT:
+      size = sizeof(GHOST_XrPose);
+      break;
+    case XR_VIBRATION_OUTPUT:
+      return action;
+  }
+  action->states = MEM_calloc_arrayN(count, size, "XrAction_States");
+  action->states_prev = MEM_calloc_arrayN(count, size, "XrAction_StatesPrev");
+
+  if (float_threshold) {
+    BLI_assert(type == XR_FLOAT_INPUT || type == XR_VECTOR2F_INPUT);
+    action->float_threshold = *float_threshold;
+    CLAMP(action->float_threshold, 0.0f, 1.0f);
+  }
+
+  action->ot = ot;
+  action->op_properties = op_properties;
+  action->op_flag = op_flag;
+
+  return action;
+}
+
+static void action_destroy(void *val)
+{
+  wmXrAction *action = val;
+
+  MEM_SAFE_FREE(action->name);
+
+  const unsigned int count = action->count_subaction_paths;
+  char **subaction_paths = action->subaction_paths;
+  if (subaction_paths) {
+    for (unsigned int i = 0; i < count; ++i) {
+      MEM_SAFE_FREE(subaction_paths[i]);
+    }
+    MEM_freeN(subaction_paths);
+  }
+
+  MEM_SAFE_FREE(action->states);
+  MEM_SAFE_FREE(action->states_prev);
+
+  MEM_freeN(action);
+}
+
+static wmXrAction *action_find(wmXrData *xr, const char *action_set_name, const char *action_name)
+{
+  return GHOST_XrGetActionCustomdata(xr->runtime->context, action_set_name, action_name);
+}
+
+bool WM_xr_action_set_create(wmXrData *xr, const char *action_set_name)
+{
+  if (action_set_find(xr, action_set_name)) {
+    return false;
+  }
+
+  wmXrActionSet *action_set = action_set_create(action_set_name);
+
+  GHOST_XrActionSetInfo info = {
+      .name = action_set_name,
+      .customdata_free_fn = action_set_destroy,
+      .customdata = action_set,
+  };
+
+  if (!GHOST_XrCreateActionSet(xr->runtime->context, &info)) {
+    return false;
+  }
+
+  return true;
+}
+
+void WM_xr_action_set_destroy(wmXrData *xr, const char *action_set_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return;
+  }
+
+  wmXrSessionState *session_state = &xr->runtime->session_state;
+
+  if (action_set == session_state->active_action_set) {
+    if (action_set->controller_pose_action) {
+      wm_xr_session_controller_data_clear(session_state);
+      action_set->controller_pose_action = NULL;
+    }
+    if (action_set->active_modal_action) {
+      action_set->active_modal_action = NULL;
+    }
+    session_state->active_action_set = NULL;
+  }
+
+  GHOST_XrDestroyActionSet(xr->runtime->context, action_set_name);
+}
+
+bool WM_xr_action_create(wmXrData *xr,
+                         const char *action_set_name,
+                         const char *action_name,
+                         eXrActionType type,
+                         unsigned int count_subaction_paths,
+                         const char **subaction_paths,
+                         const float *float_threshold,
+                         wmOperatorType *ot,
+                         IDProperty *op_properties,
+                         eXrOpFlag op_flag)
+{
+  if (action_find(xr, action_set_name, action_name)) {
+    return false;
+  }
+
+  wmXrAction *action = action_create(action_name,
+                                     type,
+                                     count_subaction_paths,
+                                     subaction_paths,
+                                     float_threshold,
+                                     ot,
+                                     op_properties,
+                                     op_flag);
+
+  GHOST_XrActionInfo info = {
+      .name = action_name,
+      .count_subaction_paths = count_subaction_paths,
+      .subaction_paths = subaction_paths,
+      .states = action->states,
+      .customdata_free_fn = action_destroy,
+      .customdata = action,
+  };
+
+  switch (type) {
+    case XR_BOOLEAN_INPUT:
+      info.type = GHOST_kXrActionTypeBooleanInput;
+      break;
+    case XR_FLOAT_INPUT:
+      info.type = GHOST_kXrActionTypeFloatInput;
+      break;
+    case XR_VECTOR2F_INPUT:
+      info.type = GHOST_kXrActionTypeVector2fInput;
+      break;
+    case XR_POSE_INPUT:
+      info.type = GHOST_kXrActionTypePoseInput;
+      break;
+    case XR_VIBRATION_OUTPUT:
+      info.type = GHOST_kXrActionTypeVibrationOutput;
+      break;
+  }
+
+  if (!GHOST_XrCreateActions(xr->runtime->context, action_set_name, 1, &info)) {
+    return false;
+  }
+
+  return true;
+}
+
+void WM_xr_action_destroy(wmXrData *xr, const char *action_set_name, const char *action_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return;
+  }
+
+  if (action_set->controller_pose_action &&
+      STREQ(action_set->controller_pose_action->name, action_name)) {
+    if (action_set == xr->runtime->session_state.active_action_set) {
+      wm_xr_session_controller_data_clear(&xr->runtime->session_state);
+    }
+    action_set->controller_pose_action = NULL;
+  }
+  if (action_set->active_modal_action &&
+      STREQ(action_set->active_modal_action->name, action_name)) {
+    action_set->active_modal_action = NULL;
+  }
+
+  wmXrAction *action = action_find(xr, action_set_name, action_name);
+  if (!action) {
+    return;
+  }
+}
+
+bool WM_xr_action_space_create(wmXrData *xr,
+                               const char *action_set_name,
+                               const char *action_name,
+                               unsigned int count_subaction_paths,
+                               const char **subaction_paths,
+                               const wmXrPose *poses)
+{
+  GHOST_XrActionSpaceInfo info = {
+      .action_name = action_name,
+      .count_subaction_paths = count_subaction_paths,
+      .subaction_paths = subaction_paths,
+  };
+
+  GHOST_XrPose *ghost_poses = MEM_malloc_arrayN(
+      count_subaction_paths, sizeof(*ghost_poses), __func__);
+  for (unsigned int i = 0; i < count_subaction_paths; ++i) {
+    const wmXrPose *pose = &poses[i];
+    GHOST_XrPose *ghost_pose = &ghost_poses[i];
+    copy_v3_v3(ghost_pose->position, pose->position);
+    copy_qt_qt(ghost_pose->orientation_quat, pose->orientation_quat);
+  }
+  info.poses = ghost_poses;
+
+  bool ret = GHOST_XrCreateActionSpaces(xr->runtime->context, action_set_name, 1, &info) ? true :
+                                                                                           false;
+  MEM_freeN(ghost_poses);
+  return ret;
+}
+
+void WM_xr_action_space_destroy(wmXrData *xr,
+                                const char *action_set_name,
+                                const char *action_name,
+                                unsigned int count_subaction_paths,
+                                const char **subaction_paths)
+{
+  GHOST_XrActionSpaceInfo info = {
+      .action_name = action_name,
+      .count_subaction_paths = count_subaction_paths,
+      .subaction_paths = subaction_paths,
+  };
+
+  GHOST_XrDestroyActionSpaces(xr->runtime->context, action_set_name, 1, &info);
+}
+
+bool WM_xr_action_binding_create(wmXrData *xr,
+                                 const char *action_set_name,
+                                 const char *profile_path,
+                                 const char *action_name,
+                                 unsigned int count_interaction_paths,
+                                 const char **interaction_paths)
+{
+  GHOST_XrActionBindingInfo binding_info = {
+      .action_name = action_name,
+      .count_interaction_paths = count_interaction_paths,
+      .interaction_paths = interaction_paths,
+  };
+
+  GHOST_XrActionProfileInfo profile_info = {
+      .profile_path = profile_path,
+      .count_bindings = 1,
+      .bindings = &binding_info,
+  };
+
+  return GHOST_XrCreateActionBindings(xr->runtime->context, action_set_name, 1, &profile_info);
+}
+
+void WM_xr_action_binding_destroy(wmXrData *xr,
+                                  const char *action_set_name,
+                                  const char *profile_path,
+                                  const char *action_name,
+                                  unsigned int count_interaction_paths,
+                                  const char **interaction_paths)
+{
+  GHOST_XrActionBindingInfo binding_info = {
+      .action_name = action_name,
+      .count_interaction_paths = count_interaction_paths,
+      .interaction_paths = interaction_paths,
+  };
+
+  GHOST_XrActionProfileInfo profile_info = {
+      .profile_path = profile_path,
+      .count_bindings = 1,
+      .bindings = &binding_info,
+  };
+
+  GHOST_XrDestroyActionBindings(xr->runtime->context, action_set_name, 1, &profile_info);
+}
+
+bool WM_xr_active_action_set_set(wmXrData *xr, const char *action_set_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return false;
+  }
+
+  {
+    /* Unset active modal action (if any). */
+    wmXrActionSet *active_action_set = xr->runtime->session_state.active_action_set;
+    if (active_action_set) {
+      wmXrAction *active_modal_action = active_action_set->active_modal_action;
+      if (active_modal_action) {
+        if (active_modal_action->active_modal_path) {
+          active_modal_action->active_modal_path = NULL;
+        }
+        active_action_set->active_modal_action = NULL;
+      }
+    }
+  }
+
+  xr->runtime->session_state.active_action_set = action_set;
+
+  if (action_set->controller_pose_action) {
+    wm_xr_session_controller_data_populate(action_set->controller_pose_action, xr);
+  }
+
+  return true;
+}
+
+bool WM_xr_controller_pose_action_set(wmXrData *xr,
+                                      const char *action_set_name,
+                                      const char *action_name)
+{
+  wmXrActionSet *action_set = action_set_find(xr, action_set_name);
+  if (!action_set) {
+    return false;
+  }
+
+  wmXrAction *action = action_find(xr, action_set_name, action_name);
+  if (!action) {
+    return false;
+  }
+
+  action_set->controller_pose_action = action;
+
+  if (action_set == xr->runtime->session_state.active_action_set) {
+    wm_xr_session_controller_data_populate(action, xr);
+  }
+
+  return true;
+}
+
+bool WM_xr_action_state_get(const wmXrData *xr,
+                            const char *action_set_name,
+                            const char *action_name,
+                            const char *subaction_path,
+                            wmXrActionState *r_state)
+{
+  const wmXrAction *action = action_find((wmXrData *)xr, action_set_name, action_name);
+  if (!action) {
+    return false;
+  }
+
+  BLI_assert(action->type == (eXrActionType)r_state->type);
+
+  /* Find the action state corresponding to the subaction path. */
+  for (unsigned int i = 0; i < action->count_subaction_paths; ++i) {
+    if (STREQ(subaction_path, action->subaction_paths[i])) {
+      switch ((eXrActionType)r_state->type) {
+        case XR_BOOLEAN_INPUT:
+          r_state->state_boolean = ((bool *)action->states)[i];
+          break;
+        case XR_FLOAT_INPUT:
+          r_state->state_float = ((float *)action->states)[i];
+          break;
+        case XR_VECTOR2F_INPUT:
+          copy_v2_v2(r_state->state_vector2f, ((float(*)[2])action->states)[i]);
+          break;
+        case XR_POSE_INPUT: {
+          const GHOST_XrPose *pose = &((GHOST_XrPose *)action->states)[i];
+          copy_v3_v3(r_state->state_pose.position, pose->position);
+          copy_qt_qt(r_state->state_pose.orientation_quat, pose->orientation_quat);
+          break;
+        }
+        case XR_VIBRATION_OUTPUT:
+          BLI_assert_unreachable();
+          break;
+      }
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool WM_xr_haptic_action_apply(wmXrData *xr,
+                               const char *action_set_name,
+                               const char *action_name,
+                               const long long *duration,
+                               const float *frequency,
+                               const float *amplitude)
+{
+  return GHOST_XrApplyHapticAction(
+             xr->runtime->context, action_set_name, action_name, duration, frequency, amplitude) ?
+             true :
+             false;
+}
+
+void WM_xr_haptic_action_stop(wmXrData *xr, const char *action_set_name, const char *action_name)
+{
+  GHOST_XrStopHapticAction(xr->runtime->context, action_set_name, action_name);
+}
+
+/** \} */ /* XR-Action API */
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_draw.c b/source/blender/windowmanager/xr/intern/wm_xr_draw.c
index cc4a7e41e82..1f722855696 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_draw.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr_draw.c
@@ -45,6 +45,12 @@ void wm_xr_pose_to_viewmat(const GHOST_XrPose *pose, float r_viewmat[4][4])
   translate_m4(r_viewmat, -pose->position[0], -pose->position[1], -pose->position[2]);
 }
 
+void wm_xr_controller_pose_to_mat(const GHOST_XrPose *pose, float r_mat[4][4])
+{
+  quat_to_mat4(r_mat, pose->orientation_quat);
+  copy_v3_v3(r_mat[3], pose->position);
+}
+
 static void wm_xr_draw_matrices_create(const wmXrDrawData *draw_data,
                                        const GHOST_XrDrawViewInfo *draw_view,
                                        const XrSessionSettings *session_settings,
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_intern.h b/source/blender/windowmanager/xr/intern/wm_xr_intern.h
index 25e3da3ffb4..9bf63be61dd 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_intern.h
+++ b/source/blender/windowmanager/xr/intern/wm_xr_intern.h
@@ -24,6 +24,21 @@
 
 #include "wm_xr.h"
 
+struct wmXrActionSet;
+
+typedef struct wmXrControllerData {
+  /** OpenXR path identifier. Length is dependent on OpenXR's XR_MAX_PATH_LENGTH (256).
+  This subaction path will later be combined with a component path, and that combined path should
+  also have a max of XR_MAX_PATH_LENGTH (e.g. subaction_path = /user/hand/left, component_path =
+  /input/trigger/value, interaction_path = /user/hand/left/input/trigger/value).
+  */
+  char subaction_path[64];
+  /** Last known controller pose (in world space) stored for queries. */
+  GHOST_XrPose pose;
+  /** The last known controller matrix, calculated from above's controller pose. */
+  float mat[4][4];
+} wmXrControllerData;
+
 typedef struct wmXrSessionState {
   bool is_started;
 
@@ -39,11 +54,23 @@ typedef struct wmXrSessionState {
   Object *prev_base_pose_object;
   /** Copy of XrSessionSettings.flag created on the last draw call, stored to detect changes. */
   int prev_settings_flag;
+  /** Copy of wmXrDrawData.base_pose. */
+  GHOST_XrPose prev_base_pose;
+  /** Copy of GHOST_XrDrawViewInfo.local_pose. */
+  GHOST_XrPose prev_local_pose;
   /** Copy of wmXrDrawData.eye_position_ofs. */
   float prev_eye_position_ofs[3];
 
   bool force_reset_to_base_pose;
   bool is_view_data_set;
+
+  /** Last known controller data. */
+  wmXrControllerData controllers[2];
+
+  /** The currently active action set that will be updated on calls to
+   * wm_xr_session_actions_update(). If NULL, all action sets will be treated as active and
+   * updated. */
+  struct wmXrActionSet *active_action_set;
 } wmXrSessionState;
 
 typedef struct wmXrRuntimeData {
@@ -79,6 +106,40 @@ typedef struct wmXrDrawData {
   float eye_position_ofs[3]; /* Local/view space. */
 } wmXrDrawData;
 
+typedef struct wmXrAction {
+  char *name;
+  eXrActionType type;
+  unsigned int count_subaction_paths;
+  char **subaction_paths;
+  /** States for each subaction path. */
+  void *states;
+  /** Previous states, stored to determine XR events. */
+  void *states_prev;
+
+  /** Input threshold for float/vector2f actions. */
+  float float_threshold;
+
+  /** The currently active subaction path (if any) for modal actions. */
+  char **active_modal_path;
+
+  /** Operator to be called on XR events. */
+  struct wmOperatorType *ot;
+  IDProperty *op_properties;
+  eXrOpFlag op_flag;
+} wmXrAction;
+
+typedef struct wmXrActionSet {
+  char *name;
+
+  /** The XR pose action that determines the controller
+   * transforms. This is usually identified by the OpenXR path "/grip/pose" or "/aim/pose",
+   * although it could differ depending on the specification and hardware. */
+  wmXrAction *controller_pose_action;
+
+  /** The currently active modal action (if any). */
+  wmXrAction *active_modal_action;
+} wmXrActionSet;
+
 wmXrRuntimeData *wm_xr_runtime_data_create(void);
 void wm_xr_runtime_data_free(wmXrRuntimeData **runtime);
 
@@ -95,5 +156,12 @@ bool wm_xr_session_surface_offscreen_ensure(wmXrSurfaceData *surface_data,
 void *wm_xr_session_gpu_binding_context_create(void);
 void wm_xr_session_gpu_binding_context_destroy(GHOST_ContextHandle context);
 
+void wm_xr_session_actions_init(wmXrData *xr);
+void wm_xr_session_actions_update(wmXrData *xr);
+void wm_xr_session_controller_data_populate(const wmXrAction *controller_pose_action,
+                                            wmXrData *xr);
+void wm_xr_session_controller_data_clear(wmXrSessionState *state);
+
 void wm_xr_pose_to_viewmat(const GHOST_XrPose *pose, float r_viewmat[4][4]);
+void wm_xr_controller_pose_to_mat(const GHOST_XrPose *pose, float r_mat[4][4]);
 void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata);
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_session.c b/source/blender/windowmanager/xr/intern/wm_xr_session.c
index b9ef40e3398..1ddbe228e05 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_session.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr_session.c
@@ -18,7 +18,9 @@
  * \ingroup wm
  */
 
+#include "BKE_callbacks.h"
 #include "BKE_context.h"
+#include "BKE_global.h"
 #include "BKE_main.h"
 #include "BKE_scene.h"
 
@@ -49,11 +51,24 @@ static CLG_LogRef LOG = {"wm.xr"};
 
 /* -------------------------------------------------------------------- */
 
+static void wm_xr_session_create_cb(void)
+{
+  Main *bmain = G_MAIN;
+  wmWindowManager *wm = bmain->wm.first;
+  wmXrData *xr_data = &wm->xr;
+
+  /* Get action set data from Python. */
+  BKE_callback_exec_null(bmain, BKE_CB_EVT_XR_SESSION_START_PRE);
+
+  wm_xr_session_actions_init(xr_data);
+}
+
 static void wm_xr_session_exit_cb(void *customdata)
 {
   wmXrData *xr_data = customdata;
 
   xr_data->runtime->session_state.is_started = false;
+
   if (xr_data->runtime->exit_fn) {
     xr_data->runtime->exit_fn(xr_data);
   }
@@ -65,6 +80,10 @@ static void wm_xr_session_exit_cb(void *customdata)
 static void wm_xr_session_begin_info_create(wmXrData *xr_data,
                                             GHOST_XrSessionBeginInfo *r_begin_info)
 {
+  /* Callback for when the session is created. This is needed to create and bind OpenXR actions
+   * after the session is created but before it is started. */
+  r_begin_info->create_fn = wm_xr_session_create_cb;
+
   /* WM-XR exit function, does some own stuff and calls callback passed to wm_xr_session_toggle(),
    * to allow external code to execute its own session-exit logic. */
   r_begin_info->exit_fn = wm_xr_session_exit_cb;
@@ -289,6 +308,7 @@ void wm_xr_session_draw_data_update(const wmXrSessionState *state,
 /**
  * Update information that is only stored for external state queries. E.g. for Python API to
  * request the current (as in, last known) viewer pose.
+ * Controller data and action sets will be updated separately via wm_xr_session_actions_update().
  */
 void wm_xr_session_state_update(const XrSessionSettings *settings,
                                 const wmXrDrawData *draw_data,
@@ -322,6 +342,8 @@ void wm_xr_session_state_update(const XrSessionSettings *settings,
                                         DEFAULT_SENSOR_WIDTH);
 
   copy_v3_v3(state->prev_eye_position_ofs, draw_data->eye_position_ofs);
+  memcpy(&state->prev_base_pose, &draw_data->base_pose, sizeof(state->prev_base_pose));
+  memcpy(&state->prev_local_pose, &draw_view->local_pose, sizeof(state->prev_local_pose));
   state->prev_settings_flag = settings->flag;
   state->prev_base_pose_type = settings->base_pose_type;
   state->prev_base_pose_object = settings->base_pose_object;
@@ -373,6 +395,132 @@ bool WM_xr_session_state_viewer_pose_matrix_info_get(const wmXrData *xr,
   return true;
 }
 
+bool WM_xr_session_state_controller_pose_location_get(const wmXrData *xr,
+                                                      unsigned int subaction_idx,
+                                                      float r_location[3])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set ||
+      subaction_idx >= ARRAY_SIZE(xr->runtime->session_state.controllers)) {
+    zero_v3(r_location);
+    return false;
+  }
+
+  copy_v3_v3(r_location, xr->runtime->session_state.controllers[subaction_idx].pose.position);
+  return true;
+}
+
+bool WM_xr_session_state_controller_pose_rotation_get(const wmXrData *xr,
+                                                      unsigned int subaction_idx,
+                                                      float r_rotation[4])
+{
+  if (!WM_xr_session_is_ready(xr) || !xr->runtime->session_state.is_view_data_set ||
+      subaction_idx >= ARRAY_SIZE(xr->runtime->session_state.controllers)) {
+    unit_qt(r_rotation);
+    return false;
+  }
+
+  copy_v4_v4(r_rotation,
+             xr->runtime->session_state.controllers[subaction_idx].pose.orientation_quat);
+  return true;
+}
+
+/* -------------------------------------------------------------------- */
+/** \name XR-Session Actions
+ *
+ * XR action processing and event dispatching.
+ *
+ * \{ */
+
+void wm_xr_session_actions_init(wmXrData *xr)
+{
+  if (!xr->runtime) {
+    return;
+  }
+
+  GHOST_XrAttachActionSets(xr->runtime->context);
+}
+
+static void wm_xr_session_controller_mats_update(const XrSessionSettings *settings,
+                                                 const wmXrAction *controller_pose_action,
+                                                 wmXrSessionState *state)
+{
+  const unsigned int count = (unsigned int)min_ii(
+      (int)controller_pose_action->count_subaction_paths, (int)ARRAY_SIZE(state->controllers));
+
+  float view_ofs[3];
+  float base_inv[4][4];
+  float tmp[4][4];
+
+  zero_v3(view_ofs);
+  if ((settings->flag & XR_SESSION_USE_POSITION_TRACKING) == 0) {
+    add_v3_v3(view_ofs, state->prev_local_pose.position);
+  }
+
+  wm_xr_pose_to_viewmat(&state->prev_base_pose, base_inv);
+  invert_m4(base_inv);
+
+  for (unsigned int i = 0; i < count; ++i) {
+    wmXrControllerData *controller = &state->controllers[i];
+
+    /* Calculate controller matrix in world space. */
+    wm_xr_controller_pose_to_mat(&((GHOST_XrPose *)controller_pose_action->states)[i], tmp);
+
+    /* Apply eye position and base pose offsets. */
+    sub_v3_v3(tmp[3], view_ofs);
+    mul_m4_m4m4(controller->mat, base_inv, tmp);
+
+    /* Save final pose. */
+    mat4_to_loc_quat(
+        controller->pose.position, controller->pose.orientation_quat, controller->mat);
+  }
+}
+
+void wm_xr_session_actions_update(wmXrData *xr)
+{
+  if (!xr->runtime) {
+    return;
+  }
+
+  GHOST_XrContextHandle xr_context = xr->runtime->context;
+  wmXrSessionState *state = &xr->runtime->session_state;
+  wmXrActionSet *active_action_set = state->active_action_set;
+
+  int ret = GHOST_XrSyncActions(xr_context, active_action_set ? active_action_set->name : NULL);
+  if (!ret) {
+    return;
+  }
+
+  /* Only update controller mats for active action set. */
+  if (active_action_set) {
+    if (active_action_set->controller_pose_action) {
+      wm_xr_session_controller_mats_update(
+          &xr->session_settings, active_action_set->controller_pose_action, state);
+    }
+  }
+}
+
+void wm_xr_session_controller_data_populate(const wmXrAction *controller_pose_action, wmXrData *xr)
+{
+  wmXrSessionState *state = &xr->runtime->session_state;
+
+  const unsigned int count = (unsigned int)min_ii(
+      (int)ARRAY_SIZE(state->controllers), (int)controller_pose_action->count_subaction_paths);
+
+  for (unsigned int i = 0; i < count; ++i) {
+    wmXrControllerData *c = &state->controllers[i];
+    strcpy(c->subaction_path, controller_pose_action->subaction_paths[i]);
+    memset(&c->pose, 0, sizeof(c->pose));
+    zero_m4(c->mat);
+  }
+}
+
+void wm_xr_session_controller_data_clear(wmXrSessionState *state)
+{
+  memset(state->controllers, 0, sizeof(state->controllers));
+}
+
+/** \} */ /* XR-Session Actions */
+
 /* -------------------------------------------------------------------- */
 /** \name XR-Session Surface
  *