Fix T32214: Wireframe edge select fails with verts behind the view

This resolves a long standing bug in edge selection
(picking, circle, box & lasso).

Now when one of the edges vertices fails to project into screen space,
the edge is clipped by the viewport to calculate an on-screen location
that can be used instead.

This isn't default as it may be important for the on the screen location
not to be clipped by the viewport.

1 files changed, 167 insertions, 39 deletions

diff --git a/source/blender/editors/space_view3d/view3d_iterators.c b/source/blender/editors/space_view3d/view3d_iterators.c
index 190cc787eea..72606e472b9 100644
--- a/source/blender/editors/space_view3d/view3d_iterators.c
+++ b/source/blender/editors/space_view3d/view3d_iterators.c
@@ -50,6 +50,55 @@
 #include "ED_screen.h"
 #include "ED_view3d.h"
 
+/* -------------------------------------------------------------------- */
+/** \name Clipping Plane Utility
+ *
+ * Calculate clipping planes to use when #V3D_PROJ_TEST_CLIP_CONTENT is enabled.
+ * \{ */
+
+/**
+ * Calculate clip planes from the viewpoint using `clip_flag`
+ * to detect which planes should be applied (maximum 6).
+ *
+ * \return The number of planes written into `planes`.
+ */
+static int content_planes_from_clip_flag(const ARegion *region,
+                                         const Object *ob,
+                                         const eV3DProjTest clip_flag,
+                                         float planes[6][4])
+{
+  float *clip_xmin = NULL, *clip_xmax = NULL;
+  float *clip_ymin = NULL, *clip_ymax = NULL;
+  float *clip_zmin = NULL, *clip_zmax = NULL;
+
+  int planes_len = 0;
+
+  if (clip_flag & V3D_PROJ_TEST_CLIP_NEAR) {
+    clip_zmin = planes[planes_len++];
+  }
+  if (clip_flag & V3D_PROJ_TEST_CLIP_FAR) {
+    clip_zmax = planes[planes_len++];
+  }
+  if (clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
+    /* The order in `planes` doesn't matter as all planes are looped over. */
+    clip_xmin = planes[planes_len++];
+    clip_xmax = planes[planes_len++];
+    clip_ymin = planes[planes_len++];
+    clip_ymax = planes[planes_len++];
+  }
+
+  BLI_assert(planes_len <= 6);
+  if (planes_len != 0) {
+    RegionView3D *rv3d = region->regiondata;
+    float projmat[4][4];
+    ED_view3d_ob_project_mat_get(rv3d, ob, projmat);
+    planes_from_projmat(projmat, clip_xmin, clip_xmax, clip_ymin, clip_ymax, clip_zmin, clip_zmax);
+  }
+  return planes_len;
+}
+
+/** \} */
+
 typedef struct foreachScreenObjectVert_userData {
   void (*func)(void *userData, MVert *mv, const float screen_co_b[2], int index);
   void *userData;
@@ -73,8 +122,16 @@ typedef struct foreachScreenEdge_userData {
                int index);
   void *userData;
   ViewContext vc;
-  rctf win_rect; /* copy of: vc.region->winx/winy, use for faster tests, minx/y will always be 0 */
   eV3DProjTest clip_flag;
+
+  rctf win_rect; /* copy of: vc.region->winx/winy, use for faster tests, minx/y will always be 0 */
+
+  /**
+   * Clip plans defined by the the view bounds,
+   * use when #V3D_PROJ_TEST_CLIP_CONTENT is enabled.
+   */
+  float content_planes[6][4];
+  int content_planes_len;
 } foreachScreenEdge_userData;
 
 typedef struct foreachScreenFace_userData {
@@ -120,6 +177,7 @@ void meshobject_foreachScreenVert(
     void *userData,
     eV3DProjTest clip_flag)
 {
+  BLI_assert((clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) == 0);
   foreachScreenObjectVert_userData data;
   Mesh *me;
 
@@ -191,6 +249,76 @@ void mesh_foreachScreenVert(
 
 /* ------------------------------------------------------------------------ */
 
+/**
+ * Edge projection is more involved since part of the edge may be behind the view
+ * or extend beyond the far limits. In the case of single points, these can be ignored.
+ * However it just may still be visible on screen, so constrained the edge to planes
+ * defined by the port to ensure both ends of the edge can be projected, see T32214.
+ *
+ * \note This is unrelated to #V3D_PROJ_TEST_CLIP_BB which must be checked separately.
+ */
+static bool mesh_foreachScreenEdge_shared_project_and_test(const ARegion *region,
+                                                           const float v0co[3],
+                                                           const float v1co[3],
+                                                           const eV3DProjTest clip_flag,
+                                                           const rctf *win_rect,
+                                                           const float content_planes[][4],
+                                                           const int content_planes_len,
+                                                           /* Output. */
+                                                           float r_screen_co_a[2],
+                                                           float r_screen_co_b[2])
+{
+  /* Clipping already handled, no need to check in projection. */
+  eV3DProjTest clip_flag_nowin = clip_flag & ~V3D_PROJ_TEST_CLIP_WIN;
+
+  const eV3DProjStatus status_a = ED_view3d_project_float_object(
+      region, v0co, r_screen_co_a, clip_flag_nowin);
+  const eV3DProjStatus status_b = ED_view3d_project_float_object(
+      region, v1co, r_screen_co_b, clip_flag_nowin);
+
+  if ((status_a == V3D_PROJ_RET_OK) && (status_b == V3D_PROJ_RET_OK)) {
+    if (clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
+      if (!BLI_rctf_isect_segment(win_rect, r_screen_co_a, r_screen_co_b)) {
+        return false;
+      }
+    }
+  }
+  else {
+    if (content_planes_len == 0) {
+      return false;
+    }
+
+    /* Both too near, ignore. */
+    if ((status_a & V3D_PROJ_TEST_CLIP_NEAR) && (status_b & V3D_PROJ_TEST_CLIP_NEAR)) {
+      return false;
+    }
+
+    /* Both too far, ignore. */
+    if ((status_a & V3D_PROJ_TEST_CLIP_FAR) && (status_b & V3D_PROJ_TEST_CLIP_FAR)) {
+      return false;
+    }
+
+    /* Simple cases have been ruled out, clip by viewport planes, then re-project. */
+    float v0co_clip[3], v1co_clip[3];
+    if (!clip_segment_v3_plane_n(
+            v0co, v1co, content_planes, content_planes_len, v0co_clip, v1co_clip)) {
+      return false;
+    }
+
+    if ((ED_view3d_project_float_object(region, v0co_clip, r_screen_co_a, clip_flag_nowin) !=
+         V3D_PROJ_RET_OK) ||
+        (ED_view3d_project_float_object(region, v1co_clip, r_screen_co_b, clip_flag_nowin) !=
+         V3D_PROJ_RET_OK)) {
+      return false;
+    }
+
+    /* No need for #V3D_PROJ_TEST_CLIP_WIN check here,
+     * clipping the segment by planes handle this. */
+  }
+
+  return true;
+}
+
 static void mesh_foreachScreenEdge__mapFunc(void *userData,
                                             int index,
                                             const float v0co[3],
@@ -202,25 +330,19 @@ static void mesh_foreachScreenEdge__mapFunc(void *userData,
     return;
   }
 
-  float screen_co_a[2];
-  float screen_co_b[2];
-  eV3DProjTest clip_flag_nowin = data->clip_flag & ~V3D_PROJ_TEST_CLIP_WIN;
-
-  if (ED_view3d_project_float_object(data->vc.region, v0co, screen_co_a, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
-    return;
-  }
-  if (ED_view3d_project_float_object(data->vc.region, v1co, screen_co_b, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
+  float screen_co_a[2], screen_co_b[2];
+  if (!mesh_foreachScreenEdge_shared_project_and_test(data->vc.region,
+                                                      v0co,
+                                                      v1co,
+                                                      data->clip_flag,
+                                                      &data->win_rect,
+                                                      data->content_planes,
+                                                      data->content_planes_len,
+                                                      screen_co_a,
+                                                      screen_co_b)) {
     return;
   }
 
-  if (data->clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
-    if (!BLI_rctf_isect_segment(&data->win_rect, screen_co_a, screen_co_b)) {
-      return;
-    }
-  }
-
   data->func(data->userData, eed, screen_co_a, screen_co_b, index);
 }
 
@@ -255,6 +377,14 @@ void mesh_foreachScreenEdge(ViewContext *vc,
     ED_view3d_clipping_local(vc->rv3d, vc->obedit->obmat); /* for local clipping lookups */
   }
 
+  if (clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) {
+    data.content_planes_len = content_planes_from_clip_flag(
+        vc->region, vc->obedit, clip_flag, data.content_planes);
+  }
+  else {
+    data.content_planes_len = 0;
+  }
+
   BM_mesh_elem_table_ensure(vc->em->bm, BM_EDGE);
   BKE_mesh_foreach_mapped_edge(me, mesh_foreachScreenEdge__mapFunc, &data);
 }
@@ -278,35 +408,24 @@ static void mesh_foreachScreenEdge_clip_bb_segment__mapFunc(void *userData,
 
   BLI_assert(data->clip_flag & V3D_PROJ_TEST_CLIP_BB);
 
-  float v0co_clip[3];
-  float v1co_clip[3];
-
+  float v0co_clip[3], v1co_clip[3];
   if (!clip_segment_v3_plane_n(v0co, v1co, data->vc.rv3d->clip_local, 4, v0co_clip, v1co_clip)) {
     return;
   }
 
-  float screen_co_a[2];
-  float screen_co_b[2];
-
-  /* Clipping already handled, no need to check in projection. */
-  eV3DProjTest clip_flag_nowin = data->clip_flag &
-                                 ~(V3D_PROJ_TEST_CLIP_WIN | V3D_PROJ_TEST_CLIP_BB);
-
-  if (ED_view3d_project_float_object(data->vc.region, v0co_clip, screen_co_a, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
-    return;
-  }
-  if (ED_view3d_project_float_object(data->vc.region, v1co_clip, screen_co_b, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
+  float screen_co_a[2], screen_co_b[2];
+  if (!mesh_foreachScreenEdge_shared_project_and_test(data->vc.region,
+                                                      v0co_clip,
+                                                      v1co_clip,
+                                                      data->clip_flag,
+                                                      &data->win_rect,
+                                                      data->content_planes,
+                                                      data->content_planes_len,
+                                                      screen_co_a,
+                                                      screen_co_b)) {
     return;
   }
 
-  if (data->clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
-    if (!BLI_rctf_isect_segment(&data->win_rect, screen_co_a, screen_co_b)) {
-      return;
-    }
-  }
-
   data->func(data->userData, eed, screen_co_a, screen_co_b, index);
 }
 
@@ -341,6 +460,14 @@ void mesh_foreachScreenEdge_clip_bb_segment(ViewContext *vc,
   data.userData = userData;
   data.clip_flag = clip_flag;
 
+  if (clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) {
+    data.content_planes_len = content_planes_from_clip_flag(
+        vc->region, vc->obedit, clip_flag, data.content_planes);
+  }
+  else {
+    data.content_planes_len = 0;
+  }
+
   BM_mesh_elem_table_ensure(vc->em->bm, BM_EDGE);
 
   if ((clip_flag & V3D_PROJ_TEST_CLIP_BB) && (vc->rv3d->clipbb != NULL)) {
@@ -380,6 +507,7 @@ void mesh_foreachScreenFace(
     void *userData,
     const eV3DProjTest clip_flag)
 {
+  BLI_assert((clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) == 0);
   foreachScreenFace_userData data;
 
   Mesh *me = editbmesh_get_eval_cage_from_orig(