Fix 'GPU_matrix_unproject_3fv' not working with out-of-bounds points

To solve this, the unproject code was redone in order to simplify and optimize.

4 files changed, 46 insertions, 92 deletions

diff --git a/source/blender/editors/space_view3d/view3d_project.c b/source/blender/editors/space_view3d/view3d_project.c
index d926ea84e0f..88efc530484 100644
--- a/source/blender/editors/space_view3d/view3d_project.c
+++ b/source/blender/editors/space_view3d/view3d_project.c
@@ -843,7 +843,7 @@ bool ED_view3d_unproject_v3(
   const int viewport[4] = {0, 0, region->winx, region->winy};
   const float region_co[3] = {regionx, regiony, regionz};
 
-  return GPU_matrix_unproject_3fv(region_co, rv3d->viewmat, rv3d->winmat, viewport, world);
+  return GPU_matrix_unproject_3fv(region_co, rv3d->viewinv, rv3d->winmat, viewport, world);
 }
 
 /** \} */
diff --git a/source/blender/gpu/GPU_matrix.h b/source/blender/gpu/GPU_matrix.h
index e073263f352..edf16f04349 100644
--- a/source/blender/gpu/GPU_matrix.h
+++ b/source/blender/gpu/GPU_matrix.h
@@ -131,15 +131,11 @@ void GPU_matrix_project_2fv(const float world[3],
                             float r_win[2]);
 
 bool GPU_matrix_unproject_3fv(const float win[3],
-                              const float model[4][4],
+                              const float model_inverted[4][4],
                               const float proj[4][4],
                               const int view[4],
                               float r_world[3]);
 
-void GPU_matrix_unproject_3fv_with_precalc(const struct GPUMatrixUnproject_Precalc *unproj_precalc,
-                                           const float win[3],
-                                           float r_world[3]);
-
 /* 2D Projection Matrix */
 
 void GPU_matrix_ortho_2d_set(float left, float right, float bottom, float top);
diff --git a/source/blender/gpu/intern/gpu_matrix.cc b/source/blender/gpu/intern/gpu_matrix.cc
index efa04568401..e277dda3812 100644
--- a/source/blender/gpu/intern/gpu_matrix.cc
+++ b/source/blender/gpu/intern/gpu_matrix.cc
@@ -513,93 +513,55 @@ void GPU_matrix_project_2fv(const float world[3],
   win[1] = view[1] + (view[3] * (v[1] + 1)) * 0.5f;
 }
 
-/**
- * The same result could be obtained as follows:
- *
- * \code{.c}
- * float projinv[4][4];
- * invert_m4_m4(projinv, projmat);
- * co[0] = 2 * co[0] - 1;
- * co[1] = 2 * co[1] - 1;
- * co[2] = 2 * co[2] - 1;
- * mul_project_m4_v3(projinv, co);
- * \endcode
- *
- * But that solution loses much precision.
- * Therefore, get the same result without inverting the matrix.
- */
-static void gpu_mul_invert_projmat_m4_unmapped_v3_with_precalc(
-    const struct GPUMatrixUnproject_Precalc *precalc, float co[3])
-{
-  /* 'precalc->dims' is the result of 'projmat_dimensions(proj, ...)'. */
-  co[0] = (float)scalenormd(precalc->dims.xmin, precalc->dims.xmax, co[0]);
-  co[1] = (float)scalenormd(precalc->dims.ymin, precalc->dims.ymax, co[1]);
-
-  if (precalc->is_persp) {
-    co[2] = (precalc->dims.zmax * precalc->dims.zmin) /
-            (precalc->dims.zmax + co[2] * (precalc->dims.zmin - precalc->dims.zmax));
-    co[0] *= co[2] / precalc->dims.zmin;
-    co[1] *= co[2] / precalc->dims.zmin;
-  }
-  else {
-    co[2] = (float)scalenormd(precalc->dims.zmin, precalc->dims.zmax, co[2]);
-  }
-  co[2] *= -1;
-}
-
-bool GPU_matrix_unproject_precalc(struct GPUMatrixUnproject_Precalc *precalc,
-                                  const float model[4][4],
-                                  const float proj[4][4],
-                                  const int view[4])
-{
-  precalc->is_persp = proj[3][3] == 0.0f;
-  projmat_dimensions_db(proj,
-                        &precalc->dims.xmin,
-                        &precalc->dims.xmax,
-                        &precalc->dims.ymin,
-                        &precalc->dims.ymax,
-                        &precalc->dims.zmin,
-                        &precalc->dims.zmax);
-  if (isinf(precalc->dims.zmax)) {
-    /* We cannot retrieve the actual value of the clip_end.
-     * Use `FLT_MAX` to avoid NAN's. */
-    precalc->dims.zmax = FLT_MAX;
-  }
-  for (int i = 0; i < 4; i++) {
-    precalc->view[i] = (float)view[i];
-  }
-  if (!invert_m4_m4(precalc->model_inverted, model)) {
-    unit_m4(precalc->model_inverted);
-    return false;
-  }
-  return true;
-}
-
-void GPU_matrix_unproject_3fv_with_precalc(const struct GPUMatrixUnproject_Precalc *precalc,
-                                           const float win[3],
-                                           float r_world[3])
-{
-  float in[3] = {
-      (win[0] - precalc->view[0]) / precalc->view[2],
-      (win[1] - precalc->view[1]) / precalc->view[3],
-      win[2],
-  };
-  gpu_mul_invert_projmat_m4_unmapped_v3_with_precalc(precalc, in);
-  mul_v3_m4v3(r_world, precalc->model_inverted, in);
-}
-
 bool GPU_matrix_unproject_3fv(const float win[3],
-                              const float model[4][4],
+                              const float model_inverted[4][4],
                               const float proj[4][4],
                               const int view[4],
                               float r_world[3])
 {
-  struct GPUMatrixUnproject_Precalc precalc;
-  if (!GPU_matrix_unproject_precalc(&precalc, model, proj, view)) {
-    zero_v3(r_world);
+  zero_v3(r_world);
+  float in[3] = {
+      2 * ((win[0] - view[0]) / view[2]) - 1.0f,
+      2 * ((win[1] - view[1]) / view[3]) - 1.0f,
+      2 * win[2] - 1.0f,
+  };
+
+  /**
+   * The same result could be obtained as follows:
+   *
+   * \code{.c}
+   * float projinv[4][4];
+   * invert_m4_m4(projinv, projview);
+   * copy_v3_v3(r_world, in);
+   * mul_project_m4_v3(projinv, r_world);
+   * \endcode
+   *
+   * But that solution loses much precision.
+   * Therefore, get the same result without inverting the project view matrix.
+   */
+
+  float out[3];
+  const bool is_persp = proj[3][3] == 0.0f;
+  if (is_persp) {
+    out[2] = proj[3][2] / (proj[2][2] + in[2]);
+    if (isinf(out[2])) {
+      out[2] = FLT_MAX;
+    }
+    out[0] = out[2] * ((proj[2][0] + in[0]) / proj[0][0]);
+    out[1] = out[2] * ((proj[2][1] + in[1]) / proj[1][1]);
+    out[2] *= -1;
+  }
+  else {
+    out[0] = (-proj[3][0] + in[0]) / proj[0][0];
+    out[1] = (-proj[3][1] + in[1]) / proj[1][1];
+    out[2] = (-proj[3][2] + in[2]) / proj[2][2];
+  }
+
+  if (!is_finite_v3(out)) {
     return false;
   }
-  GPU_matrix_unproject_3fv_with_precalc(&precalc, win, r_world);
+  
+  mul_v3_m4v3(r_world, model_inverted, out);
   return true;
 }
 
diff --git a/source/blender/windowmanager/gizmo/intern/wm_gizmo_map.c b/source/blender/windowmanager/gizmo/intern/wm_gizmo_map.c
index 5ec26a7b208..b90e288776e 100644
--- a/source/blender/windowmanager/gizmo/intern/wm_gizmo_map.c
+++ b/source/blender/windowmanager/gizmo/intern/wm_gizmo_map.c
@@ -616,11 +616,7 @@ static int gizmo_find_intersected_3d_intern(wmGizmo **visible_gizmos,
     const int viewport[4] = {0, 0, region->winx, region->winy};
     float co_3d_origin[3];
 
-    /* Avoid multiple calculations. */
-    struct GPUMatrixUnproject_Precalc unproj_precalc;
-    GPU_matrix_unproject_precalc(&unproj_precalc, rv3d->viewmat, rv3d->winmat, viewport);
-
-    GPU_matrix_unproject_3fv_with_precalc(&unproj_precalc, co_screen, co_3d_origin);
+    GPU_matrix_unproject_3fv(co_screen, rv3d->viewinv, rv3d->winmat, viewport, co_3d_origin);
 
     uint *buf_iter = buffer;
     int hit_found = -1;
@@ -631,7 +627,7 @@ static int gizmo_find_intersected_3d_intern(wmGizmo **visible_gizmos,
       wmGizmo *gz = visible_gizmos[buf_iter[3] >> 8];
       float co_3d[3];
       co_screen[2] = int_as_float(buf_iter[1]);
-      GPU_matrix_unproject_3fv_with_precalc(&unproj_precalc, co_screen, co_3d);
+      GPU_matrix_unproject_3fv(co_screen, rv3d->viewinv, rv3d->winmat, viewport, co_3d_origin);
       float select_bias = gz->select_bias;
       if ((gz->flag & WM_GIZMO_DRAW_NO_SCALE) == 0) {
         select_bias *= gz->scale_final;