XR: Color Depth Adjustments

This addresses reduced visibility of scenes (as displayed in the VR
headset) that can result from the 8-bit color depth format currently
used for XR swapchain images.

By switching to a swapchain format with higher color depth (RGB10_A2,
RGBA16, RGBA16F) for supported runtimes, visibility in VR should be
noticeably improved.

However, current limitations are lack of support for these higher
color depth formats by some XR runtimes, especially for OpenGL.

Also important to note that GPU_offscreen_create() now explicitly
takes in the texture format (eGPUTextureFormat) instead of a
"high_bitdepth" boolean.

Reviewed By: Julian Eisel, Clément Foucault

Differential Revision: http://developer.blender.org/D9842

8 files changed, 135 insertions, 18 deletions

diff --git a/intern/ghost/GHOST_Types.h b/intern/ghost/GHOST_Types.h
index fb19b9535ad..e46f712cb64 100644
--- a/intern/ghost/GHOST_Types.h
+++ b/intern/ghost/GHOST_Types.h
@@ -669,6 +669,14 @@ typedef struct {
   void *exit_customdata;
 } GHOST_XrSessionBeginInfo;
 
+/** Texture format for XR swapchain. */
+typedef enum GHOST_TXrSwapchainFormat {
+  GHOST_kXrSwapchainFormatRGBA8,
+  GHOST_kXrSwapchainFormatRGBA16,
+  GHOST_kXrSwapchainFormatRGBA16F,
+  GHOST_kXrSwapchainFormatRGB10_A2,
+} GHOST_TXrSwapchainFormat;
+
 typedef struct GHOST_XrDrawViewInfo {
   int ofsx, ofsy;
   int width, height;
@@ -681,6 +689,7 @@ typedef struct GHOST_XrDrawViewInfo {
     float angle_up, angle_down;
   } fov;
 
+  GHOST_TXrSwapchainFormat swapchain_format;
   /** Set if the buffer should be submitted with a SRGB transfer applied. */
   char expects_srgb_buffer;
 
diff --git a/intern/ghost/intern/GHOST_ContextD3D.cpp b/intern/ghost/intern/GHOST_ContextD3D.cpp
index ad948578d53..73f6c12e100 100644
--- a/intern/ghost/intern/GHOST_ContextD3D.cpp
+++ b/intern/ghost/intern/GHOST_ContextD3D.cpp
@@ -132,6 +132,7 @@ class GHOST_SharedOpenGLResource {
                              ID3D11DeviceContext *device_ctx,
                              unsigned int width,
                              unsigned int height,
+                             DXGI_FORMAT format,
                              ID3D11RenderTargetView *render_target = nullptr)
       : m_device(device), m_device_ctx(device_ctx), m_cur_width(width), m_cur_height(height)
   {
@@ -144,7 +145,7 @@ class GHOST_SharedOpenGLResource {
 
       texDesc.Width = width;
       texDesc.Height = height;
-      texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
+      texDesc.Format = format;
       texDesc.SampleDesc.Count = 1;
       texDesc.ArraySize = 1;
       texDesc.MipLevels = 1;
@@ -321,7 +322,10 @@ class GHOST_SharedOpenGLResource {
 };
 
 GHOST_SharedOpenGLResource *GHOST_ContextD3D::createSharedOpenGLResource(
-    unsigned int width, unsigned int height, ID3D11RenderTargetView *render_target)
+    unsigned int width,
+    unsigned int height,
+    DXGI_FORMAT format,
+    ID3D11RenderTargetView *render_target)
 {
   if (!(WGL_NV_DX_interop && WGL_NV_DX_interop2)) {
     fprintf(stderr,
@@ -330,14 +334,15 @@ GHOST_SharedOpenGLResource *GHOST_ContextD3D::createSharedOpenGLResource(
     return nullptr;
   }
   GHOST_SharedOpenGLResource *shared_res = new GHOST_SharedOpenGLResource(
-      m_device, m_device_ctx, width, height, render_target);
+      m_device, m_device_ctx, width, height, format, render_target);
 
   return shared_res;
 }
 GHOST_SharedOpenGLResource *GHOST_ContextD3D::createSharedOpenGLResource(unsigned int width,
-                                                                         unsigned int height)
+                                                                         unsigned int height,
+                                                                         DXGI_FORMAT format)
 {
-  return createSharedOpenGLResource(width, height, nullptr);
+  return createSharedOpenGLResource(width, height, format, nullptr);
 }
 
 void GHOST_ContextD3D::disposeSharedOpenGLResource(GHOST_SharedOpenGLResource *shared_res)
diff --git a/intern/ghost/intern/GHOST_ContextD3D.h b/intern/ghost/intern/GHOST_ContextD3D.h
index 8b9537ca439..c18c9d3c286 100644
--- a/intern/ghost/intern/GHOST_ContextD3D.h
+++ b/intern/ghost/intern/GHOST_ContextD3D.h
@@ -106,9 +106,13 @@ class GHOST_ContextD3D : public GHOST_Context {
   }
 
   class GHOST_SharedOpenGLResource *createSharedOpenGLResource(
-      unsigned int width, unsigned int height, ID3D11RenderTargetView *render_target);
+      unsigned int width,
+      unsigned int height,
+      DXGI_FORMAT format,
+      ID3D11RenderTargetView *render_target);
   class GHOST_SharedOpenGLResource *createSharedOpenGLResource(unsigned int width,
-                                                               unsigned int height);
+                                                               unsigned int height,
+                                                               DXGI_FORMAT format);
   void disposeSharedOpenGLResource(class GHOST_SharedOpenGLResource *shared_res);
   GHOST_TSuccess blitFromOpenGLContext(class GHOST_SharedOpenGLResource *shared_res,
                                        unsigned int width,
diff --git a/intern/ghost/intern/GHOST_IXrGraphicsBinding.h b/intern/ghost/intern/GHOST_IXrGraphicsBinding.h
index a7339158dc4..bfdf0cac633 100644
--- a/intern/ghost/intern/GHOST_IXrGraphicsBinding.h
+++ b/intern/ghost/intern/GHOST_IXrGraphicsBinding.h
@@ -60,6 +60,7 @@ class GHOST_IXrGraphicsBinding {
                                         std::string *r_requirement_info) const = 0;
   virtual void initFromGhostContext(class GHOST_Context &ghost_ctx) = 0;
   virtual std::optional<int64_t> chooseSwapchainFormat(const std::vector<int64_t> &runtime_formats,
+                                                       GHOST_TXrSwapchainFormat &r_format,
                                                        bool &r_is_rgb_format) const = 0;
   virtual std::vector<XrSwapchainImageBaseHeader *> createSwapchainImages(
       uint32_t image_count) = 0;
diff --git a/intern/ghost/intern/GHOST_XrGraphicsBinding.cpp b/intern/ghost/intern/GHOST_XrGraphicsBinding.cpp
index dd0205ea867..70567e02cb9 100644
--- a/intern/ghost/intern/GHOST_XrGraphicsBinding.cpp
+++ b/intern/ghost/intern/GHOST_XrGraphicsBinding.cpp
@@ -38,6 +38,7 @@
 #  include "GHOST_SystemWin32.h"
 #endif
 #include "GHOST_C-api.h"
+#include "GHOST_XrException.h"
 #include "GHOST_Xr_intern.h"
 
 #include "GHOST_IXrGraphicsBinding.h"
@@ -160,16 +161,41 @@ class GHOST_XrGraphicsBindingOpenGL : public GHOST_IXrGraphicsBinding {
   }
 
   std::optional<int64_t> chooseSwapchainFormat(const std::vector<int64_t> &runtime_formats,
+                                               GHOST_TXrSwapchainFormat &r_format,
                                                bool &r_is_srgb_format) const override
   {
     std::vector<int64_t> gpu_binding_formats = {
+        GL_RGB10_A2,
+        GL_RGBA16,
+        GL_RGBA16F,
         GL_RGBA8,
         GL_SRGB8_ALPHA8,
     };
 
     std::optional result = choose_swapchain_format_from_candidates(gpu_binding_formats,
                                                                    runtime_formats);
-    r_is_srgb_format = result ? (*result == GL_SRGB8_ALPHA8) : false;
+    if (result) {
+      switch (*result) {
+        case GL_RGB10_A2:
+          r_format = GHOST_kXrSwapchainFormatRGB10_A2;
+          break;
+        case GL_RGBA16:
+          r_format = GHOST_kXrSwapchainFormatRGBA16;
+          break;
+        case GL_RGBA16F:
+          r_format = GHOST_kXrSwapchainFormatRGBA16F;
+          break;
+        case GL_RGBA8:
+        case GL_SRGB8_ALPHA8:
+          r_format = GHOST_kXrSwapchainFormatRGBA8;
+          break;
+      }
+      r_is_srgb_format = (*result == GL_SRGB8_ALPHA8);
+    }
+    else {
+      r_format = GHOST_kXrSwapchainFormatRGBA8;
+      r_is_srgb_format = false;
+    }
 
     return result;
   }
@@ -228,6 +254,33 @@ class GHOST_XrGraphicsBindingOpenGL : public GHOST_IXrGraphicsBinding {
 };
 
 #ifdef WIN32
+static void ghost_format_to_dx_format(GHOST_TXrSwapchainFormat ghost_format,
+                                      bool expects_srgb_buffer,
+                                      DXGI_FORMAT &r_dx_format)
+{
+  r_dx_format = DXGI_FORMAT_UNKNOWN;
+
+  switch (ghost_format) {
+    case GHOST_kXrSwapchainFormatRGBA8:
+      r_dx_format = expects_srgb_buffer ? DXGI_FORMAT_R8G8B8A8_UNORM_SRGB :
+                                          DXGI_FORMAT_R8G8B8A8_UNORM;
+      break;
+    case GHOST_kXrSwapchainFormatRGBA16:
+      r_dx_format = DXGI_FORMAT_R16G16B16A16_UNORM;
+      break;
+    case GHOST_kXrSwapchainFormatRGBA16F:
+      r_dx_format = DXGI_FORMAT_R16G16B16A16_FLOAT;
+      break;
+    case GHOST_kXrSwapchainFormatRGB10_A2:
+      r_dx_format = DXGI_FORMAT_R10G10B10A2_UNORM;
+      break;
+  }
+
+  if (r_dx_format == DXGI_FORMAT_UNKNOWN) {
+    throw GHOST_XrException("No supported DirectX swapchain format found.");
+  }
+}
+
 class GHOST_XrGraphicsBindingD3D : public GHOST_IXrGraphicsBinding {
  public:
   GHOST_XrGraphicsBindingD3D(GHOST_Context &ghost_ctx)
@@ -284,16 +337,48 @@ class GHOST_XrGraphicsBindingD3D : public GHOST_IXrGraphicsBinding {
   }
 
   std::optional<int64_t> chooseSwapchainFormat(const std::vector<int64_t> &runtime_formats,
+                                               GHOST_TXrSwapchainFormat &r_format,
                                                bool &r_is_srgb_format) const override
   {
     std::vector<int64_t> gpu_binding_formats = {
-        DXGI_FORMAT_R8G8B8A8_UNORM,
-        DXGI_FORMAT_R8G8B8A8_UNORM_SRGB,
+#  if 0 /* RGB10A2 doesn't seem to work with Oculus headsets, so move it after RGB16AF for the \
+           time being. */
+        DXGI_FORMAT_R10G10B10A2_UNORM,
+#  endif
+      DXGI_FORMAT_R16G16B16A16_UNORM,
+      DXGI_FORMAT_R16G16B16A16_FLOAT,
+#  if 1
+      DXGI_FORMAT_R10G10B10A2_UNORM,
+#  endif
+      DXGI_FORMAT_R8G8B8A8_UNORM,
+      DXGI_FORMAT_R8G8B8A8_UNORM_SRGB,
     };
 
     std::optional result = choose_swapchain_format_from_candidates(gpu_binding_formats,
                                                                    runtime_formats);
-    r_is_srgb_format = result ? (*result == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB) : false;
+    if (result) {
+      switch (*result) {
+        case DXGI_FORMAT_R10G10B10A2_UNORM:
+          r_format = GHOST_kXrSwapchainFormatRGB10_A2;
+          break;
+        case DXGI_FORMAT_R16G16B16A16_UNORM:
+          r_format = GHOST_kXrSwapchainFormatRGBA16;
+          break;
+        case DXGI_FORMAT_R16G16B16A16_FLOAT:
+          r_format = GHOST_kXrSwapchainFormatRGBA16F;
+          break;
+        case DXGI_FORMAT_R8G8B8A8_UNORM:
+        case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB:
+          r_format = GHOST_kXrSwapchainFormatRGBA8;
+          break;
+      }
+      r_is_srgb_format = (*result == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB);
+    }
+    else {
+      r_format = GHOST_kXrSwapchainFormatRGBA8;
+      r_is_srgb_format = false;
+    }
+
     return result;
   }
 
@@ -334,14 +419,18 @@ class GHOST_XrGraphicsBindingD3D : public GHOST_IXrGraphicsBinding {
 
     m_ghost_ctx->m_device->CreateRenderTargetView(d3d_swapchain_image.texture, &rtv_desc, &rtv);
     if (!m_shared_resource) {
+      DXGI_FORMAT format;
+      ghost_format_to_dx_format(draw_info.swapchain_format, draw_info.expects_srgb_buffer, format);
       m_shared_resource = m_ghost_ctx->createSharedOpenGLResource(
-          draw_info.width, draw_info.height, rtv);
+          draw_info.width, draw_info.height, format, rtv);
     }
     m_ghost_ctx->blitFromOpenGLContext(m_shared_resource, draw_info.width, draw_info.height);
 #  else
     if (!m_shared_resource) {
-      m_shared_resource = m_ghost_d3d_ctx->createSharedOpenGLResource(draw_info.width,
-                                                                      draw_info.height);
+      DXGI_FORMAT format;
+      ghost_format_to_dx_format(draw_info.swapchain_format, draw_info.expects_srgb_buffer, format);
+      m_shared_resource = m_ghost_d3d_ctx->createSharedOpenGLResource(
+          draw_info.width, draw_info.height, format);
     }
     m_ghost_d3d_ctx->blitFromOpenGLContext(m_shared_resource, draw_info.width, draw_info.height);
 
diff --git a/intern/ghost/intern/GHOST_XrSession.cpp b/intern/ghost/intern/GHOST_XrSession.cpp
index 4cab22ee676..8b0320ef358 100644
--- a/intern/ghost/intern/GHOST_XrSession.cpp
+++ b/intern/ghost/intern/GHOST_XrSession.cpp
@@ -422,6 +422,7 @@ void GHOST_XrSession::drawView(GHOST_XrSwapchain &swapchain,
 
   assert(view_idx < 256);
   draw_view_info.view_idx = (char)view_idx;
+  draw_view_info.swapchain_format = swapchain.getFormat();
   draw_view_info.expects_srgb_buffer = swapchain.isBufferSRGB();
   draw_view_info.ofsx = r_proj_layer_view.subImage.imageRect.offset.x;
   draw_view_info.ofsy = r_proj_layer_view.subImage.imageRect.offset.y;
diff --git a/intern/ghost/intern/GHOST_XrSwapchain.cpp b/intern/ghost/intern/GHOST_XrSwapchain.cpp
index 9973d99cc37..f89b7227ab1 100644
--- a/intern/ghost/intern/GHOST_XrSwapchain.cpp
+++ b/intern/ghost/intern/GHOST_XrSwapchain.cpp
@@ -67,8 +67,8 @@ GHOST_XrSwapchain::GHOST_XrSwapchain(GHOST_IXrGraphicsBinding &gpu_binding,
            "Failed to get swapchain image formats.");
   assert(swapchain_formats.size() == format_count);
 
-  std::optional chosen_format = gpu_binding.chooseSwapchainFormat(swapchain_formats,
-                                                                  m_is_srgb_buffer);
+  std::optional chosen_format = gpu_binding.chooseSwapchainFormat(
+      swapchain_formats, m_format, m_is_srgb_buffer);
   if (!chosen_format) {
     throw GHOST_XrException(
         "Error: No format matching OpenXR runtime supported swapchain formats found.");
@@ -97,6 +97,7 @@ GHOST_XrSwapchain::GHOST_XrSwapchain(GHOST_XrSwapchain &&other)
     : m_oxr(std::move(other.m_oxr)),
       m_image_width(other.m_image_width),
       m_image_height(other.m_image_height),
+      m_format(other.m_format),
       m_is_srgb_buffer(other.m_is_srgb_buffer)
 {
   /* Prevent xrDestroySwapchain call for the moved out item. */
@@ -134,7 +135,12 @@ void GHOST_XrSwapchain::updateCompositionLayerProjectViewSubImage(XrSwapchainSub
   r_sub_image.imageRect.extent = {m_image_width, m_image_height};
 }
 
-bool GHOST_XrSwapchain::isBufferSRGB()
+GHOST_TXrSwapchainFormat GHOST_XrSwapchain::getFormat() const
+{
+  return m_format;
+}
+
+bool GHOST_XrSwapchain::isBufferSRGB() const
 {
   return m_is_srgb_buffer;
 }
diff --git a/intern/ghost/intern/GHOST_XrSwapchain.h b/intern/ghost/intern/GHOST_XrSwapchain.h
index 33a1c17b993..0c6592e2db6 100644
--- a/intern/ghost/intern/GHOST_XrSwapchain.h
+++ b/intern/ghost/intern/GHOST_XrSwapchain.h
@@ -37,10 +37,12 @@ class GHOST_XrSwapchain {
 
   void updateCompositionLayerProjectViewSubImage(XrSwapchainSubImage &r_sub_image);
 
-  bool isBufferSRGB();
+  GHOST_TXrSwapchainFormat getFormat() const;
+  bool isBufferSRGB() const;
 
  private:
   std::unique_ptr<OpenXRSwapchainData> m_oxr; /* Could use stack, but PImpl is preferable. */
   int32_t m_image_width, m_image_height;
+  GHOST_TXrSwapchainFormat m_format;
   bool m_is_srgb_buffer = false;
 };