XR: Support for Varjo OpenXR extensions

This adds support for two Varjo specific OpenXR vendor extensions:
1) XR_VARJO_QUAD_VIEWS
2) XR_VARJO_FOVEATED_RENDERING

Together these enable human eye resolution rendering on supported
devices (currently mainly Varjo XR-3 and VR-3).

In addition, there's a detection for Varjo OpenXR runtime.

This has been tested on real Varjo XR-3 hardware and Varjo Simulator
and confirmed to function correctly. Foveation works, and the views are
rendered correctly for all the four views.

Reviewed By: Peter Kim, Julian Eisel

Differential Revision: https://developer.blender.org/D12229

3 files changed, 97 insertions, 4 deletions

diff --git a/intern/ghost/intern/GHOST_XrContext.cpp b/intern/ghost/intern/GHOST_XrContext.cpp
index a7498e9f91f..88da39914e7 100644
--- a/intern/ghost/intern/GHOST_XrContext.cpp
+++ b/intern/ghost/intern/GHOST_XrContext.cpp
@@ -135,7 +135,8 @@ void GHOST_XrContext::storeInstanceProperties()
       {"Monado(XRT) by Collabora et al", OPENXR_RUNTIME_MONADO},
       {"Oculus", OPENXR_RUNTIME_OCULUS},
       {"SteamVR/OpenXR", OPENXR_RUNTIME_STEAMVR},
-      {"Windows Mixed Reality Runtime", OPENXR_RUNTIME_WMR}};
+      {"Windows Mixed Reality Runtime", OPENXR_RUNTIME_WMR},
+      {"Varjo OpenXR Runtime", OPENXR_RUNTIME_VARJO}};
   decltype(runtime_map)::const_iterator runtime_map_iter;
 
   m_oxr->instance_properties.type = XR_TYPE_INSTANCE_PROPERTIES;
@@ -415,6 +416,12 @@ void GHOST_XrContext::getExtensionsToEnable(
   try_ext.push_back(XR_HTC_VIVE_COSMOS_CONTROLLER_INTERACTION_EXTENSION_NAME);
   try_ext.push_back(XR_HUAWEI_CONTROLLER_INTERACTION_EXTENSION_NAME);
 
+  /* Varjo quad view extension. */
+  try_ext.push_back(XR_VARJO_QUAD_VIEWS_EXTENSION_NAME);
+
+  /* Varjo foveated extension. */
+  try_ext.push_back(XR_VARJO_FOVEATED_RENDERING_EXTENSION_NAME);
+
   r_ext_names.reserve(try_ext.size() + graphics_binding_types.size());
 
   /* Add graphics binding extensions (may be multiple ones, we'll settle for one to use later, once
@@ -613,4 +620,11 @@ bool GHOST_XrContext::isDebugTimeMode() const
   return m_debug_time;
 }
 
+bool GHOST_XrContext::isExtensionEnabled(const char *ext) const
+{
+  bool contains = std::find(m_enabled_extensions.begin(), m_enabled_extensions.end(), ext) !=
+                  m_enabled_extensions.end();
+  return contains;
+}
+
 /** \} */ /* Ghost Internal Accessors and Mutators */
diff --git a/intern/ghost/intern/GHOST_XrContext.h b/intern/ghost/intern/GHOST_XrContext.h
index f29d7349f7e..26fc374b957 100644
--- a/intern/ghost/intern/GHOST_XrContext.h
+++ b/intern/ghost/intern/GHOST_XrContext.h
@@ -51,6 +51,7 @@ enum GHOST_TXrOpenXRRuntimeID {
   OPENXR_RUNTIME_OCULUS,
   OPENXR_RUNTIME_STEAMVR,
   OPENXR_RUNTIME_WMR, /* Windows Mixed Reality */
+  OPENXR_RUNTIME_VARJO,
 
   OPENXR_RUNTIME_UNKNOWN
 };
@@ -94,6 +95,8 @@ class GHOST_XrContext : public GHOST_IXrContext {
   bool isDebugMode() const;
   bool isDebugTimeMode() const;
 
+  bool isExtensionEnabled(const char *ext) const;
+
  private:
   static GHOST_XrErrorHandlerFn s_error_handler;
   static void *s_error_handler_customdata;
diff --git a/intern/ghost/intern/GHOST_XrSession.cpp b/intern/ghost/intern/GHOST_XrSession.cpp
index a08b2d6045a..cd930c8328b 100644
--- a/intern/ghost/intern/GHOST_XrSession.cpp
+++ b/intern/ghost/intern/GHOST_XrSession.cpp
@@ -41,10 +41,13 @@ struct OpenXRSessionData {
   XrSession session = XR_NULL_HANDLE;
   XrSessionState session_state = XR_SESSION_STATE_UNKNOWN;
 
-  /* Only stereo rendering supported now. */
-  const XrViewConfigurationType view_type = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO;
+  /* Use stereo rendering by default. */
+  XrViewConfigurationType view_type = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO;
+  bool foveation_supported = false;
+
   XrSpace reference_space;
   XrSpace view_space;
+  XrSpace combined_eye_space;
   std::vector<XrView> views;
   std::vector<GHOST_XrSwapchain> swapchains;
 
@@ -58,6 +61,9 @@ struct GHOST_XrDrawInfo {
   std::chrono::high_resolution_clock::time_point frame_begin_time;
   /* Time previous frames took for rendering (in ms). */
   std::list<double> last_frame_times;
+
+  /* Whether foveation is active for the frame. */
+  bool foveation_active;
 };
 
 /* -------------------------------------------------------------------- */
@@ -82,6 +88,9 @@ GHOST_XrSession::~GHOST_XrSession()
   if (m_oxr->view_space != XR_NULL_HANDLE) {
     CHECK_XR_ASSERT(xrDestroySpace(m_oxr->view_space));
   }
+  if (m_oxr->combined_eye_space != XR_NULL_HANDLE) {
+    CHECK_XR_ASSERT(xrDestroySpace(m_oxr->combined_eye_space));
+  }
   if (m_oxr->session != XR_NULL_HANDLE) {
     CHECK_XR_ASSERT(xrDestroySession(m_oxr->session));
   }
@@ -189,6 +198,13 @@ static void create_reference_spaces(OpenXRSessionData &oxr, const GHOST_XrPose &
   create_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW;
   CHECK_XR(xrCreateReferenceSpace(oxr.session, &create_info, &oxr.view_space),
            "Failed to create view reference space.");
+
+  /* Foveation reference spaces. */
+  if (oxr.foveation_supported) {
+    create_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_COMBINED_EYE_VARJO;
+    CHECK_XR(xrCreateReferenceSpace(oxr.session, &create_info, &oxr.combined_eye_space),
+             "Failed to create combined eye reference space.");
+  }
 }
 
 void GHOST_XrSession::start(const GHOST_XrSessionBeginInfo *begin_info)
@@ -292,9 +308,19 @@ GHOST_XrSession::LifeExpectancy GHOST_XrSession::handleStateChangeEvent(
 
 void GHOST_XrSession::prepareDrawing()
 {
+  assert(m_context->getInstance() != XR_NULL_HANDLE);
+
   std::vector<XrViewConfigurationView> view_configs;
   uint32_t view_count;
 
+  /* Attempt to use quad view if supported. */
+  if (m_context->isExtensionEnabled(XR_VARJO_QUAD_VIEWS_EXTENSION_NAME)) {
+    m_oxr->view_type = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_QUAD_VARJO;
+  }
+
+  m_oxr->foveation_supported = m_context->isExtensionEnabled(
+      XR_VARJO_FOVEATED_RENDERING_EXTENSION_NAME);
+
   CHECK_XR(
       xrEnumerateViewConfigurationViews(
           m_context->getInstance(), m_oxr->system_id, m_oxr->view_type, 0, &view_count, nullptr),
@@ -306,7 +332,36 @@ void GHOST_XrSession::prepareDrawing()
                                              view_configs.size(),
                                              &view_count,
                                              view_configs.data()),
-           "Failed to get count of view configurations.");
+           "Failed to get view configurations.");
+
+  /* If foveated rendering is used, query the foveated views. */
+  if (m_oxr->foveation_supported) {
+    std::vector<XrFoveatedViewConfigurationViewVARJO> request_foveated_config{
+        view_count, {XR_TYPE_FOVEATED_VIEW_CONFIGURATION_VIEW_VARJO, nullptr, XR_TRUE}};
+
+    auto foveated_views = std::vector<XrViewConfigurationView>(view_count,
+                                                               {XR_TYPE_VIEW_CONFIGURATION_VIEW});
+
+    for (uint32_t i = 0; i < view_count; i++) {
+      foveated_views[i].next = &request_foveated_config[i];
+    }
+    CHECK_XR(xrEnumerateViewConfigurationViews(m_context->getInstance(),
+                                               m_oxr->system_id,
+                                               m_oxr->view_type,
+                                               view_configs.size(),
+                                               &view_count,
+                                               foveated_views.data()),
+             "Failed to get foveated view configurations.");
+
+    /* Ensure swapchains have correct size even when foveation is being used. */
+    for (uint32_t i = 0; i < view_count; i++) {
+      view_configs[i].recommendedImageRectWidth = std::max(
+          view_configs[i].recommendedImageRectWidth, foveated_views[i].recommendedImageRectWidth);
+      view_configs[i].recommendedImageRectHeight = std::max(
+          view_configs[i].recommendedImageRectHeight,
+          foveated_views[i].recommendedImageRectHeight);
+    }
+  }
 
   for (const XrViewConfigurationView &view_config : view_configs) {
     m_oxr->swapchains.emplace_back(*m_gpu_binding, m_oxr->session, view_config);
@@ -327,6 +382,20 @@ void GHOST_XrSession::beginFrameDrawing()
   CHECK_XR(xrWaitFrame(m_oxr->session, &wait_info, &frame_state),
            "Failed to synchronize frame rates between Blender and the device.");
 
+  /* Check if we have foveation available for the current frame. */
+  m_draw_info->foveation_active = false;
+  if (m_oxr->foveation_supported) {
+    XrSpaceLocation render_gaze_location{XR_TYPE_SPACE_LOCATION};
+    CHECK_XR(xrLocateSpace(m_oxr->combined_eye_space,
+                           m_oxr->view_space,
+                           frame_state.predictedDisplayTime,
+                           &render_gaze_location),
+             "Failed to locate combined eye space.");
+
+    m_draw_info->foveation_active = (render_gaze_location.locationFlags &
+                                     XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT) != 0;
+  }
+
   CHECK_XR(xrBeginFrame(m_oxr->session, &begin_info),
            "Failed to submit frame rendering start state.");
 
@@ -442,6 +511,8 @@ XrCompositionLayerProjection GHOST_XrSession::drawLayer(
     std::vector<XrCompositionLayerProjectionView> &r_proj_layer_views, void *draw_customdata)
 {
   XrViewLocateInfo viewloc_info = {XR_TYPE_VIEW_LOCATE_INFO};
+  XrViewLocateFoveatedRenderingVARJO foveated_info{
+      XR_TYPE_VIEW_LOCATE_FOVEATED_RENDERING_VARJO, nullptr, true};
   XrViewState view_state = {XR_TYPE_VIEW_STATE};
   XrCompositionLayerProjection layer = {XR_TYPE_COMPOSITION_LAYER_PROJECTION};
   XrSpaceLocation view_location{XR_TYPE_SPACE_LOCATION};
@@ -451,6 +522,10 @@ XrCompositionLayerProjection GHOST_XrSession::drawLayer(
   viewloc_info.displayTime = m_draw_info->frame_state.predictedDisplayTime;
   viewloc_info.space = m_oxr->reference_space;
 
+  if (m_draw_info->foveation_active) {
+    viewloc_info.next = &foveated_info;
+  }
+
   CHECK_XR(xrLocateViews(m_oxr->session,
                          &viewloc_info,
                          &view_state,
@@ -458,6 +533,7 @@ XrCompositionLayerProjection GHOST_XrSession::drawLayer(
                          &view_count,
                          m_oxr->views.data()),
            "Failed to query frame view and projection state.");
+
   assert(m_oxr->swapchains.size() == view_count);
 
   CHECK_XR(