XR: View adjustments for variable viewer scale

This adjusts some calculations and visibility flags for XR viewports in
order to account for a possible scale factor in the XR view matrix.

This scale factor can be introduced via the XR session settings base
scale, which allows a viewer to begin their session at a specific
reference scale, or the XR session state navigation scale, which allows
a viewer to adjust their scale relative to the reference scale during
the session.

Reviewed by Severin as part of D11501, but requested to be committed
separately.

4 files changed, 22 insertions, 4 deletions

diff --git a/source/blender/draw/engines/overlay/overlay_grid.c b/source/blender/draw/engines/overlay/overlay_grid.c
index 31c8ed9d664..4a551c4dec5 100644
--- a/source/blender/draw/engines/overlay/overlay_grid.c
+++ b/source/blender/draw/engines/overlay/overlay_grid.c
@@ -200,6 +200,15 @@ void OVERLAY_grid_init(OVERLAY_Data *vedata)
   shd->grid_distance = dist / 2.0f;
 
   ED_view3d_grid_steps(scene, v3d, rv3d, shd->grid_steps);
+
+  if ((v3d->flag & (V3D_XR_SESSION_SURFACE | V3D_XR_SESSION_MIRROR)) != 0) {
+    /* The calculations for the grid parameters assume that the view matrix has no scale component,
+     * which may not be correct if the user is "shrunk" or "enlarged" by zooming in or out.
+     * Therefore, we need to compensate the values here. */
+    float viewinvscale = len_v3(
+        viewinv[0]); /* Assumption is uniform scaling (all column vectors are of same length). */
+    shd->grid_distance *= viewinvscale;
+  }
 }
 
 void OVERLAY_grid_cache_init(OVERLAY_Data *vedata)
diff --git a/source/blender/draw/intern/draw_manager_data.c b/source/blender/draw/intern/draw_manager_data.c
index f96bd474aec..5d3e3db866f 100644
--- a/source/blender/draw/intern/draw_manager_data.c
+++ b/source/blender/draw/intern/draw_manager_data.c
@@ -1684,10 +1684,12 @@ static void draw_frustum_bound_sphere_calc(const BoundBox *bbox,
     bsphere->center[0] = farcenter[0] * z / e;
     bsphere->center[1] = farcenter[1] * z / e;
     bsphere->center[2] = z;
-    bsphere->radius = len_v3v3(bsphere->center, farpoint);
 
-    /* Transform to world space. */
-    mul_m4_v3(viewinv, bsphere->center);
+    /* For XR, the view matrix may contain a scale factor. Then, transforming only the center
+     * into world space after calculating the radius will result in incorrect behavior. */
+    mul_m4_v3(viewinv, bsphere->center); /* Transform to world space. */
+    mul_m4_v3(viewinv, farpoint);
+    bsphere->radius = len_v3v3(bsphere->center, farpoint);
   }
 }
 
diff --git a/source/blender/editors/space_view3d/view3d_draw.c b/source/blender/editors/space_view3d/view3d_draw.c
index fe347e89600..fceb6553cab 100644
--- a/source/blender/editors/space_view3d/view3d_draw.c
+++ b/source/blender/editors/space_view3d/view3d_draw.c
@@ -347,6 +347,8 @@ static void view3d_xr_mirror_setup(const wmWindowManager *wm,
                      (wm->xr.session_settings.draw_flags & V3D_OFSDRAW_XR_SHOW_CUSTOM_OVERLAYS) !=
                          0,
                      V3D_XR_SHOW_CUSTOM_OVERLAYS);
+  /* Hide navigation gizmo since it gets distorted if the view matrix has a scale factor. */
+  v3d->gizmo_flag |= V3D_GIZMO_HIDE_NAVIGATE;
 
   /* Reset overridden View3D data. */
   v3d->lens = lens_old;
diff --git a/source/blender/editors/space_view3d/view3d_view.c b/source/blender/editors/space_view3d/view3d_view.c
index f5da7c14a88..46a664f10fa 100644
--- a/source/blender/editors/space_view3d/view3d_view.c
+++ b/source/blender/editors/space_view3d/view3d_view.c
@@ -1730,7 +1730,12 @@ void ED_view3d_xr_shading_update(wmWindowManager *wm, const View3D *v3d, const S
   if (v3d->runtime.flag & V3D_RUNTIME_XR_SESSION_ROOT) {
     View3DShading *xr_shading = &wm->xr.session_settings.shading;
     /* Flags that shouldn't be overridden by the 3D View shading. */
-    const int flag_copy = V3D_SHADING_WORLD_ORIENTATION;
+    int flag_copy = 0;
+    if (v3d->shading.type !=
+        OB_SOLID) { /* Don't set V3D_SHADING_WORLD_ORIENTATION for solid shading since it results
+                       in distorted lighting when the view matrix has a scale factor. */
+      flag_copy |= V3D_SHADING_WORLD_ORIENTATION;
+    }
 
     BLI_assert(WM_xr_session_exists(&wm->xr));