XR Controller Support Step 4: Controller Drawing

Addresses T77127 (Controller Drawing).

Adds VR controller visualization and custom drawing via draw
handlers. Add-ons can draw to the XR surface (headset display) and
mirror window by adding a View3D draw handler of region type 'XR' and
draw type 'POST_VIEW'.  Controller drawing and custom overlays can be
toggled individually as XR session options, which will be added in a
future update to the VR Scene Inspection add-on.

For the actual drawing, the OpenXR XR_MSFT_controller_model extension
is used to load a glTF model provided by the XR runtime. The model's
vertex data is then used to create a GPUBatch in the XR session
state. Finally, this batch is drawn via the XR surface draw handler
mentioned above.

For runtimes that do not support the controller model extension, a
a simple fallback shape (sphere) is drawn instead.

Reviewed By: Severin, fclem

Differential Revision: https://developer.blender.org/D10948

3 files changed, 278 insertions, 3 deletions

diff --git a/source/blender/windowmanager/xr/intern/wm_xr_draw.c b/source/blender/windowmanager/xr/intern/wm_xr_draw.c
index bbb73fc2007..6b3756c8178 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_draw.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr_draw.c
@@ -24,12 +24,17 @@
 
 #include <string.h>
 
+#include "BKE_context.h"
+
 #include "BLI_listbase.h"
 #include "BLI_math.h"
 
 #include "ED_view3d_offscreen.h"
 
 #include "GHOST_C-api.h"
+#include "GPU_batch_presets.h"
+#include "GPU_immediate.h"
+#include "GPU_matrix.h"
 
 #include "GPU_viewport.h"
 
@@ -153,6 +158,7 @@ void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata)
                                   winmat,
                                   settings->clip_start,
                                   settings->clip_end,
+                                  true,
                                   false,
                                   true,
                                   NULL,
@@ -172,3 +178,200 @@ void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata)
 
   wm_xr_draw_viewport_buffers_to_active_framebuffer(xr_data->runtime, surface_data, draw_view);
 }
+
+static GPUBatch *wm_xr_controller_model_batch_create(GHOST_XrContextHandle xr_context,
+                                                     const char *subaction_path)
+{
+  GHOST_XrControllerModelData model_data;
+
+  if (!GHOST_XrGetControllerModelData(xr_context, subaction_path, &model_data) ||
+      model_data.count_vertices < 1) {
+    return NULL;
+  }
+
+  GPUVertFormat format = {0};
+  GPU_vertformat_attr_add(&format, "pos", GPU_COMP_F32, 3, GPU_FETCH_FLOAT);
+  GPU_vertformat_attr_add(&format, "nor", GPU_COMP_F32, 3, GPU_FETCH_FLOAT);
+
+  GPUVertBuf *vbo = GPU_vertbuf_create_with_format(&format);
+  GPU_vertbuf_data_alloc(vbo, model_data.count_vertices);
+  void *vbo_data = GPU_vertbuf_get_data(vbo);
+  memcpy(
+      vbo_data, model_data.vertices, model_data.count_vertices * sizeof(model_data.vertices[0]));
+
+  GPUIndexBuf *ibo = NULL;
+  if (model_data.count_indices > 0 && ((model_data.count_indices % 3) == 0)) {
+    GPUIndexBufBuilder ibo_builder;
+    const unsigned int prim_len = model_data.count_indices / 3;
+    GPU_indexbuf_init(&ibo_builder, GPU_PRIM_TRIS, prim_len, model_data.count_vertices);
+    for (unsigned int i = 0; i < prim_len; ++i) {
+      const uint32_t *idx = &model_data.indices[i * 3];
+      GPU_indexbuf_add_tri_verts(&ibo_builder, idx[0], idx[1], idx[2]);
+    }
+    ibo = GPU_indexbuf_build(&ibo_builder);
+  }
+
+  return GPU_batch_create_ex(GPU_PRIM_TRIS, vbo, ibo, GPU_BATCH_OWNS_VBO | GPU_BATCH_OWNS_INDEX);
+}
+
+static void wm_xr_controller_model_draw(const XrSessionSettings *settings,
+                                        GHOST_XrContextHandle xr_context,
+                                        wmXrSessionState *state)
+{
+  GHOST_XrControllerModelData model_data;
+
+  float color[4];
+  switch (settings->controller_draw_style) {
+    case XR_CONTROLLER_DRAW_DARK:
+    case XR_CONTROLLER_DRAW_DARK_RAY:
+      color[0] = color[1] = color[2] = 0.0f, color[3] = 0.4f;
+      break;
+    case XR_CONTROLLER_DRAW_LIGHT:
+    case XR_CONTROLLER_DRAW_LIGHT_RAY:
+      color[0] = 0.422f, color[1] = 0.438f, color[2] = 0.446f, color[3] = 0.4f;
+      break;
+  }
+
+  GPU_depth_test(GPU_DEPTH_NONE);
+  GPU_blend(GPU_BLEND_ALPHA);
+
+  LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+    GPUBatch *model = controller->model;
+    if (!model) {
+      model = controller->model = wm_xr_controller_model_batch_create(xr_context,
+                                                                      controller->subaction_path);
+    }
+
+    if (model &&
+        GHOST_XrGetControllerModelData(xr_context, controller->subaction_path, &model_data) &&
+        model_data.count_components > 0) {
+      GPU_batch_program_set_builtin(model, GPU_SHADER_3D_UNIFORM_COLOR);
+      GPU_batch_uniform_4fv(model, "color", color);
+
+      GPU_matrix_push();
+      GPU_matrix_mul(controller->grip_mat);
+      for (unsigned int component_idx = 0; component_idx < model_data.count_components;
+           ++component_idx) {
+        const GHOST_XrControllerModelComponent *component = &model_data.components[component_idx];
+        GPU_matrix_push();
+        GPU_matrix_mul(component->transform);
+        GPU_batch_draw_range(model,
+                             model->elem ? component->index_offset : component->vertex_offset,
+                             model->elem ? component->index_count : component->vertex_count);
+        GPU_matrix_pop();
+      }
+      GPU_matrix_pop();
+    }
+    else {
+      /* Fallback. */
+      const float scale = 0.05f;
+      GPUBatch *sphere = GPU_batch_preset_sphere(2);
+      GPU_batch_program_set_builtin(sphere, GPU_SHADER_3D_UNIFORM_COLOR);
+      GPU_batch_uniform_4fv(sphere, "color", color);
+
+      GPU_matrix_push();
+      GPU_matrix_mul(controller->grip_mat);
+      GPU_matrix_scale_1f(scale);
+      GPU_batch_draw(sphere);
+      GPU_matrix_pop();
+    }
+  }
+}
+
+static void wm_xr_controller_aim_draw(const XrSessionSettings *settings, wmXrSessionState *state)
+{
+  bool draw_ray;
+  switch (settings->controller_draw_style) {
+    case XR_CONTROLLER_DRAW_DARK:
+    case XR_CONTROLLER_DRAW_LIGHT:
+      draw_ray = false;
+      break;
+    case XR_CONTROLLER_DRAW_DARK_RAY:
+    case XR_CONTROLLER_DRAW_LIGHT_RAY:
+      draw_ray = true;
+      break;
+  }
+
+  GPUVertFormat *format = immVertexFormat();
+  uint pos = GPU_vertformat_attr_add(format, "pos", GPU_COMP_F32, 3, GPU_FETCH_FLOAT);
+  uint col = GPU_vertformat_attr_add(format, "color", GPU_COMP_U8, 4, GPU_FETCH_INT_TO_FLOAT_UNIT);
+  immBindBuiltinProgram(GPU_SHADER_3D_POLYLINE_FLAT_COLOR);
+
+  float viewport[4];
+  GPU_viewport_size_get_f(viewport);
+  immUniform2fv("viewportSize", &viewport[2]);
+
+  immUniform1f("lineWidth", 3.0f * U.pixelsize);
+
+  if (draw_ray) {
+    const uchar color[4] = {89, 89, 255, 127};
+    const float scale = settings->clip_end;
+    float ray[3];
+
+    GPU_depth_test(GPU_DEPTH_LESS_EQUAL);
+    GPU_blend(GPU_BLEND_ALPHA);
+
+    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(&state->controllers) * 2);
+
+    LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+      const float(*mat)[4] = controller->aim_mat;
+      madd_v3_v3v3fl(ray, mat[3], mat[2], -scale);
+
+      immAttrSkip(col);
+      immVertex3fv(pos, mat[3]);
+      immAttr4ubv(col, color);
+      immVertex3fv(pos, ray);
+    }
+
+    immEnd();
+  }
+  else {
+    const uchar r[4] = {255, 51, 82, 255};
+    const uchar g[4] = {139, 220, 0, 255};
+    const uchar b[4] = {40, 144, 255, 255};
+    const float scale = 0.01f;
+    float x_axis[3], y_axis[3], z_axis[3];
+
+    GPU_depth_test(GPU_DEPTH_NONE);
+    GPU_blend(GPU_BLEND_NONE);
+
+    immBegin(GPU_PRIM_LINES, (uint)BLI_listbase_count(&state->controllers) * 6);
+
+    LISTBASE_FOREACH (wmXrController *, controller, &state->controllers) {
+      const float(*mat)[4] = controller->aim_mat;
+      madd_v3_v3v3fl(x_axis, mat[3], mat[0], scale);
+      madd_v3_v3v3fl(y_axis, mat[3], mat[1], scale);
+      madd_v3_v3v3fl(z_axis, mat[3], mat[2], scale);
+
+      immAttrSkip(col);
+      immVertex3fv(pos, mat[3]);
+      immAttr4ubv(col, r);
+      immVertex3fv(pos, x_axis);
+
+      immAttrSkip(col);
+      immVertex3fv(pos, mat[3]);
+      immAttr4ubv(col, g);
+      immVertex3fv(pos, y_axis);
+
+      immAttrSkip(col);
+      immVertex3fv(pos, mat[3]);
+      immAttr4ubv(col, b);
+      immVertex3fv(pos, z_axis);
+    }
+
+    immEnd();
+  }
+
+  immUnbindProgram();
+}
+
+void wm_xr_draw_controllers(const bContext *UNUSED(C), ARegion *UNUSED(region), void *customdata)
+{
+  wmXrData *xr = customdata;
+  const XrSessionSettings *settings = &xr->session_settings;
+  GHOST_XrContextHandle xr_context = xr->runtime->context;
+  wmXrSessionState *state = &xr->runtime->session_state;
+
+  wm_xr_controller_model_draw(settings, xr_context, state);
+  wm_xr_controller_aim_draw(settings, state);
+}
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_intern.h b/source/blender/windowmanager/xr/intern/wm_xr_intern.h
index 8fb5424b8c2..2cd0ba5c056 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_intern.h
+++ b/source/blender/windowmanager/xr/intern/wm_xr_intern.h
@@ -88,6 +88,11 @@ typedef struct wmXrViewportPair {
 typedef struct {
   /** Off-screen buffers/viewports for each view. */
   ListBase viewports; /* #wmXrViewportPair */
+
+  /** Dummy region type for controller draw callback. */
+  struct ARegionType *controller_art;
+  /** Controller draw callback handle. */
+  void *controller_draw_handle;
 } wmXrSurfaceData;
 
 typedef struct wmXrDrawData {
@@ -114,12 +119,16 @@ typedef struct wmXrController {
   /input/trigger/value, interaction_path = /user/hand/left/input/trigger/value).
   */
   char subaction_path[64];
-  /* Pose (in world space) that represents the user's hand when holding the controller.*/
+
+  /** Pose (in world space) that represents the user's hand when holding the controller. */
   GHOST_XrPose grip_pose;
   float grip_mat[4][4];
-  /* Pose (in world space) that represents the controller's aiming source. */
+  /** Pose (in world space) that represents the controller's aiming source. */
   GHOST_XrPose aim_pose;
   float aim_mat[4][4];
+
+  /** Controller model. */
+  struct GPUBatch *model;
 } wmXrController;
 
 typedef struct wmXrAction {
@@ -207,3 +216,4 @@ void wm_xr_session_controller_data_clear(wmXrSessionState *state);
 void wm_xr_pose_to_mat(const GHOST_XrPose *pose, float r_mat[4][4]);
 void wm_xr_pose_to_imat(const GHOST_XrPose *pose, float r_imat[4][4]);
 void wm_xr_draw_view(const GHOST_XrDrawViewInfo *draw_view, void *customdata);
+void wm_xr_draw_controllers(const struct bContext *C, struct ARegion *region, void *customdata);
diff --git a/source/blender/windowmanager/xr/intern/wm_xr_session.c b/source/blender/windowmanager/xr/intern/wm_xr_session.c
index 88bc5c45c91..9f53db1347c 100644
--- a/source/blender/windowmanager/xr/intern/wm_xr_session.c
+++ b/source/blender/windowmanager/xr/intern/wm_xr_session.c
@@ -24,6 +24,7 @@
 #include "BKE_idprop.h"
 #include "BKE_main.h"
 #include "BKE_scene.h"
+#include "BKE_screen.h"
 
 #include "BLI_listbase.h"
 #include "BLI_math.h"
@@ -36,9 +37,11 @@
 #include "DRW_engine.h"
 
 #include "ED_screen.h"
+#include "ED_space_api.h"
 
 #include "GHOST_C-api.h"
 
+#include "GPU_batch.h"
 #include "GPU_viewport.h"
 
 #include "MEM_guardedalloc.h"
@@ -72,7 +75,15 @@ static void wm_xr_session_create_cb(void)
 
 static void wm_xr_session_controller_data_free(wmXrSessionState *state)
 {
-  BLI_freelistN(&state->controllers);
+  ListBase *lb = &state->controllers;
+  wmXrController *c;
+
+  while ((c = BLI_pophead(lb))) {
+    if (c->model) {
+      GPU_batch_discard(c->model);
+    }
+    BLI_freelinkN(lb, c);
+  }
 }
 
 void wm_xr_session_data_free(wmXrSessionState *state)
@@ -529,6 +540,7 @@ static void wm_xr_session_controller_pose_calc(const GHOST_XrPose *raw_pose,
 static void wm_xr_session_controller_data_update(const XrSessionSettings *settings,
                                                  const wmXrAction *grip_action,
                                                  const wmXrAction *aim_action,
+                                                 GHOST_XrContextHandle xr_context,
                                                  wmXrSessionState *state)
 {
   BLI_assert(grip_action->count_subaction_paths == aim_action->count_subaction_paths);
@@ -560,6 +572,16 @@ static void wm_xr_session_controller_data_update(const XrSessionSettings *settin
                                        base_mat,
                                        &controller->aim_pose,
                                        controller->aim_mat);
+
+    if (!controller->model) {
+      /* Notify GHOST to load/continue loading the controller model data. This can be called more
+       * than once since the model may not be available from the runtime yet. The batch itself will
+       * be created in wm_xr_draw_controllers(). */
+      GHOST_XrLoadControllerModel(xr_context, controller->subaction_path);
+    }
+    else {
+      GHOST_XrUpdateControllerModelComponents(xr_context, controller->subaction_path);
+    }
   }
 }
 
@@ -1089,6 +1111,7 @@ void wm_xr_session_actions_update(wmWindowManager *wm)
       wm_xr_session_controller_data_update(&xr->session_settings,
                                            active_action_set->controller_grip_action,
                                            active_action_set->controller_aim_action,
+                                           xr_context,
                                            state);
     }
 
@@ -1125,11 +1148,33 @@ void wm_xr_session_controller_data_populate(const wmXrAction *grip_action,
 
     BLI_addtail(controllers, controller);
   }
+
+  /* Activate draw callback. */
+  if (g_xr_surface) {
+    wmXrSurfaceData *surface_data = g_xr_surface->customdata;
+    if (surface_data && !surface_data->controller_draw_handle) {
+      if (surface_data->controller_art) {
+        surface_data->controller_draw_handle = ED_region_draw_cb_activate(
+            surface_data->controller_art, wm_xr_draw_controllers, xr, REGION_DRAW_POST_VIEW);
+      }
+    }
+  }
 }
 
 void wm_xr_session_controller_data_clear(wmXrSessionState *state)
 {
   wm_xr_session_controller_data_free(state);
+
+  /* Deactivate draw callback. */
+  if (g_xr_surface) {
+    wmXrSurfaceData *surface_data = g_xr_surface->customdata;
+    if (surface_data && surface_data->controller_draw_handle) {
+      if (surface_data->controller_art) {
+        ED_region_draw_cb_exit(surface_data->controller_art, surface_data->controller_draw_handle);
+      }
+      surface_data->controller_draw_handle = NULL;
+    }
+  }
 }
 
 /** \} */ /* XR-Session Actions */
@@ -1255,6 +1300,11 @@ static void wm_xr_session_surface_free_data(wmSurface *surface)
     BLI_freelinkN(lb, vp);
   }
 
+  if (data->controller_art) {
+    BLI_freelistN(&data->controller_art->drawcalls);
+    MEM_freeN(data->controller_art);
+  }
+
   MEM_freeN(surface->customdata);
 
   g_xr_surface = NULL;
@@ -1269,6 +1319,7 @@ static wmSurface *wm_xr_session_surface_create(void)
 
   wmSurface *surface = MEM_callocN(sizeof(*surface), __func__);
   wmXrSurfaceData *data = MEM_callocN(sizeof(*data), "XrSurfaceData");
+  data->controller_art = MEM_callocN(sizeof(*(data->controller_art)), "XrControllerRegionType");
 
   surface->draw = wm_xr_session_surface_draw;
   surface->free_data = wm_xr_session_surface_free_data;
@@ -1278,6 +1329,7 @@ static wmSurface *wm_xr_session_surface_create(void)
   surface->ghost_ctx = DRW_xr_opengl_context_get();
   surface->gpu_ctx = DRW_xr_gpu_context_get();
 
+  data->controller_art->regionid = RGN_TYPE_XR;
   surface->customdata = data;
 
   g_xr_surface = surface;
@@ -1311,4 +1363,14 @@ void wm_xr_session_gpu_binding_context_destroy(GHOST_ContextHandle UNUSED(contex
   WM_main_add_notifier(NC_WM | ND_XR_DATA_CHANGED, NULL);
 }
 
+ARegionType *WM_xr_surface_controller_region_type_get(void)
+{
+  if (g_xr_surface) {
+    wmXrSurfaceData *data = g_xr_surface->customdata;
+    return data->controller_art;
+  }
+
+  return NULL;
+}
+
 /** \} */ /* XR-Session Surface */