Fix T32214: Wireframe edge select fails with verts behind the view

This resolves a long standing bug in edge selection
(picking, circle, box & lasso).

Now when one of the edges vertices fails to project into screen space,
the edge is clipped by the viewport to calculate an on-screen location
that can be used instead.

This isn't default as it may be important for the on the screen location
not to be clipped by the viewport.

4 files changed, 224 insertions, 48 deletions

diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index bdaea7d8bff..d86041aa6e8 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -196,12 +196,38 @@ typedef enum {
   V3D_PROJ_TEST_CLIP_NEAR = (1 << 2),
   V3D_PROJ_TEST_CLIP_FAR = (1 << 3),
   V3D_PROJ_TEST_CLIP_ZERO = (1 << 4),
+  /**
+   * Clip the contents of the data being iterated over.
+   * Currently this is only used to edges when projecting into screen space.
+   *
+   * Clamp the edge within the viewport limits defined by
+   * #V3D_PROJ_TEST_CLIP_WIN, #V3D_PROJ_TEST_CLIP_NEAR & #V3D_PROJ_TEST_CLIP_FAR.
+   * This resolves the problem of a visible edge having one of it's vertices
+   * behind the viewport. See: T32214.
+   *
+   * This is not default behavior as it may be important for the screen-space location
+   * of an edges vertex to represent that vertices location (instead of a location along the edge).
+   *
+   * \note Perspective views should enable #V3D_PROJ_TEST_CLIP_WIN along with
+   * #V3D_PROJ_TEST_CLIP_NEAR as the near-plane-clipped location of a point
+   * may become very large (even infinite) when projected into screen-space.
+   * Unless the that point happens to coincide with the camera's point of view.
+   *
+   * Use #V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT instead of #V3D_PROJ_TEST_CLIP_CONTENT,
+   * to avoid accidentally enabling near clipping without clipping by window bounds.
+   */
+  V3D_PROJ_TEST_CLIP_CONTENT = (1 << 5),
 } eV3DProjTest;
 
 #define V3D_PROJ_TEST_CLIP_DEFAULT \
   (V3D_PROJ_TEST_CLIP_BB | V3D_PROJ_TEST_CLIP_WIN | V3D_PROJ_TEST_CLIP_NEAR)
 #define V3D_PROJ_TEST_ALL \
-  (V3D_PROJ_TEST_CLIP_DEFAULT | V3D_PROJ_TEST_CLIP_FAR | V3D_PROJ_TEST_CLIP_ZERO)
+  (V3D_PROJ_TEST_CLIP_DEFAULT | V3D_PROJ_TEST_CLIP_FAR | V3D_PROJ_TEST_CLIP_ZERO | \
+   V3D_PROJ_TEST_CLIP_CONTENT)
+
+#define V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT \
+  (V3D_PROJ_TEST_CLIP_CONTENT | V3D_PROJ_TEST_CLIP_NEAR | V3D_PROJ_TEST_CLIP_FAR | \
+   V3D_PROJ_TEST_CLIP_WIN)
 
 /* view3d_iterators.c */
 
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 6e5dd47e282..b21645a1a5d 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -547,8 +547,10 @@ BMEdge *EDBM_edge_find_nearest_ex(ViewContext *vc,
 
       ED_view3d_init_mats_rv3d(vc->obedit, vc->rv3d);
 
-      mesh_foreachScreenEdge(
-          vc, find_nearest_edge_center__doZBuf, &data, V3D_PROJ_TEST_CLIP_DEFAULT);
+      mesh_foreachScreenEdge(vc,
+                             find_nearest_edge_center__doZBuf,
+                             &data,
+                             V3D_PROJ_TEST_CLIP_DEFAULT | V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT);
 
       *r_dist_center_px_manhattan = data.dist;
     }
@@ -601,7 +603,8 @@ BMEdge *EDBM_edge_find_nearest_ex(ViewContext *vc,
         *dist_px_manhattan_p;
 
     ED_view3d_init_mats_rv3d(vc->obedit, vc->rv3d);
-    mesh_foreachScreenEdge(vc, find_nearest_edge__doClosest, &data, clip_flag);
+    mesh_foreachScreenEdge(
+        vc, find_nearest_edge__doClosest, &data, clip_flag | V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT);
 
     hit = (data.use_cycle && data.hit_cycle.edge) ? &data.hit_cycle : &data.hit;
 
diff --git a/source/blender/editors/space_view3d/view3d_iterators.c b/source/blender/editors/space_view3d/view3d_iterators.c
index 190cc787eea..72606e472b9 100644
--- a/source/blender/editors/space_view3d/view3d_iterators.c
+++ b/source/blender/editors/space_view3d/view3d_iterators.c
@@ -50,6 +50,55 @@
 #include "ED_screen.h"
 #include "ED_view3d.h"
 
+/* -------------------------------------------------------------------- */
+/** \name Clipping Plane Utility
+ *
+ * Calculate clipping planes to use when #V3D_PROJ_TEST_CLIP_CONTENT is enabled.
+ * \{ */
+
+/**
+ * Calculate clip planes from the viewpoint using `clip_flag`
+ * to detect which planes should be applied (maximum 6).
+ *
+ * \return The number of planes written into `planes`.
+ */
+static int content_planes_from_clip_flag(const ARegion *region,
+                                         const Object *ob,
+                                         const eV3DProjTest clip_flag,
+                                         float planes[6][4])
+{
+  float *clip_xmin = NULL, *clip_xmax = NULL;
+  float *clip_ymin = NULL, *clip_ymax = NULL;
+  float *clip_zmin = NULL, *clip_zmax = NULL;
+
+  int planes_len = 0;
+
+  if (clip_flag & V3D_PROJ_TEST_CLIP_NEAR) {
+    clip_zmin = planes[planes_len++];
+  }
+  if (clip_flag & V3D_PROJ_TEST_CLIP_FAR) {
+    clip_zmax = planes[planes_len++];
+  }
+  if (clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
+    /* The order in `planes` doesn't matter as all planes are looped over. */
+    clip_xmin = planes[planes_len++];
+    clip_xmax = planes[planes_len++];
+    clip_ymin = planes[planes_len++];
+    clip_ymax = planes[planes_len++];
+  }
+
+  BLI_assert(planes_len <= 6);
+  if (planes_len != 0) {
+    RegionView3D *rv3d = region->regiondata;
+    float projmat[4][4];
+    ED_view3d_ob_project_mat_get(rv3d, ob, projmat);
+    planes_from_projmat(projmat, clip_xmin, clip_xmax, clip_ymin, clip_ymax, clip_zmin, clip_zmax);
+  }
+  return planes_len;
+}
+
+/** \} */
+
 typedef struct foreachScreenObjectVert_userData {
   void (*func)(void *userData, MVert *mv, const float screen_co_b[2], int index);
   void *userData;
@@ -73,8 +122,16 @@ typedef struct foreachScreenEdge_userData {
                int index);
   void *userData;
   ViewContext vc;
-  rctf win_rect; /* copy of: vc.region->winx/winy, use for faster tests, minx/y will always be 0 */
   eV3DProjTest clip_flag;
+
+  rctf win_rect; /* copy of: vc.region->winx/winy, use for faster tests, minx/y will always be 0 */
+
+  /**
+   * Clip plans defined by the the view bounds,
+   * use when #V3D_PROJ_TEST_CLIP_CONTENT is enabled.
+   */
+  float content_planes[6][4];
+  int content_planes_len;
 } foreachScreenEdge_userData;
 
 typedef struct foreachScreenFace_userData {
@@ -120,6 +177,7 @@ void meshobject_foreachScreenVert(
     void *userData,
     eV3DProjTest clip_flag)
 {
+  BLI_assert((clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) == 0);
   foreachScreenObjectVert_userData data;
   Mesh *me;
 
@@ -191,6 +249,76 @@ void mesh_foreachScreenVert(
 
 /* ------------------------------------------------------------------------ */
 
+/**
+ * Edge projection is more involved since part of the edge may be behind the view
+ * or extend beyond the far limits. In the case of single points, these can be ignored.
+ * However it just may still be visible on screen, so constrained the edge to planes
+ * defined by the port to ensure both ends of the edge can be projected, see T32214.
+ *
+ * \note This is unrelated to #V3D_PROJ_TEST_CLIP_BB which must be checked separately.
+ */
+static bool mesh_foreachScreenEdge_shared_project_and_test(const ARegion *region,
+                                                           const float v0co[3],
+                                                           const float v1co[3],
+                                                           const eV3DProjTest clip_flag,
+                                                           const rctf *win_rect,
+                                                           const float content_planes[][4],
+                                                           const int content_planes_len,
+                                                           /* Output. */
+                                                           float r_screen_co_a[2],
+                                                           float r_screen_co_b[2])
+{
+  /* Clipping already handled, no need to check in projection. */
+  eV3DProjTest clip_flag_nowin = clip_flag & ~V3D_PROJ_TEST_CLIP_WIN;
+
+  const eV3DProjStatus status_a = ED_view3d_project_float_object(
+      region, v0co, r_screen_co_a, clip_flag_nowin);
+  const eV3DProjStatus status_b = ED_view3d_project_float_object(
+      region, v1co, r_screen_co_b, clip_flag_nowin);
+
+  if ((status_a == V3D_PROJ_RET_OK) && (status_b == V3D_PROJ_RET_OK)) {
+    if (clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
+      if (!BLI_rctf_isect_segment(win_rect, r_screen_co_a, r_screen_co_b)) {
+        return false;
+      }
+    }
+  }
+  else {
+    if (content_planes_len == 0) {
+      return false;
+    }
+
+    /* Both too near, ignore. */
+    if ((status_a & V3D_PROJ_TEST_CLIP_NEAR) && (status_b & V3D_PROJ_TEST_CLIP_NEAR)) {
+      return false;
+    }
+
+    /* Both too far, ignore. */
+    if ((status_a & V3D_PROJ_TEST_CLIP_FAR) && (status_b & V3D_PROJ_TEST_CLIP_FAR)) {
+      return false;
+    }
+
+    /* Simple cases have been ruled out, clip by viewport planes, then re-project. */
+    float v0co_clip[3], v1co_clip[3];
+    if (!clip_segment_v3_plane_n(
+            v0co, v1co, content_planes, content_planes_len, v0co_clip, v1co_clip)) {
+      return false;
+    }
+
+    if ((ED_view3d_project_float_object(region, v0co_clip, r_screen_co_a, clip_flag_nowin) !=
+         V3D_PROJ_RET_OK) ||
+        (ED_view3d_project_float_object(region, v1co_clip, r_screen_co_b, clip_flag_nowin) !=
+         V3D_PROJ_RET_OK)) {
+      return false;
+    }
+
+    /* No need for #V3D_PROJ_TEST_CLIP_WIN check here,
+     * clipping the segment by planes handle this. */
+  }
+
+  return true;
+}
+
 static void mesh_foreachScreenEdge__mapFunc(void *userData,
                                             int index,
                                             const float v0co[3],
@@ -202,25 +330,19 @@ static void mesh_foreachScreenEdge__mapFunc(void *userData,
     return;
   }
 
-  float screen_co_a[2];
-  float screen_co_b[2];
-  eV3DProjTest clip_flag_nowin = data->clip_flag & ~V3D_PROJ_TEST_CLIP_WIN;
-
-  if (ED_view3d_project_float_object(data->vc.region, v0co, screen_co_a, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
-    return;
-  }
-  if (ED_view3d_project_float_object(data->vc.region, v1co, screen_co_b, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
+  float screen_co_a[2], screen_co_b[2];
+  if (!mesh_foreachScreenEdge_shared_project_and_test(data->vc.region,
+                                                      v0co,
+                                                      v1co,
+                                                      data->clip_flag,
+                                                      &data->win_rect,
+                                                      data->content_planes,
+                                                      data->content_planes_len,
+                                                      screen_co_a,
+                                                      screen_co_b)) {
     return;
   }
 
-  if (data->clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
-    if (!BLI_rctf_isect_segment(&data->win_rect, screen_co_a, screen_co_b)) {
-      return;
-    }
-  }
-
   data->func(data->userData, eed, screen_co_a, screen_co_b, index);
 }
 
@@ -255,6 +377,14 @@ void mesh_foreachScreenEdge(ViewContext *vc,
     ED_view3d_clipping_local(vc->rv3d, vc->obedit->obmat); /* for local clipping lookups */
   }
 
+  if (clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) {
+    data.content_planes_len = content_planes_from_clip_flag(
+        vc->region, vc->obedit, clip_flag, data.content_planes);
+  }
+  else {
+    data.content_planes_len = 0;
+  }
+
   BM_mesh_elem_table_ensure(vc->em->bm, BM_EDGE);
   BKE_mesh_foreach_mapped_edge(me, mesh_foreachScreenEdge__mapFunc, &data);
 }
@@ -278,35 +408,24 @@ static void mesh_foreachScreenEdge_clip_bb_segment__mapFunc(void *userData,
 
   BLI_assert(data->clip_flag & V3D_PROJ_TEST_CLIP_BB);
 
-  float v0co_clip[3];
-  float v1co_clip[3];
-
+  float v0co_clip[3], v1co_clip[3];
   if (!clip_segment_v3_plane_n(v0co, v1co, data->vc.rv3d->clip_local, 4, v0co_clip, v1co_clip)) {
     return;
   }
 
-  float screen_co_a[2];
-  float screen_co_b[2];
-
-  /* Clipping already handled, no need to check in projection. */
-  eV3DProjTest clip_flag_nowin = data->clip_flag &
-                                 ~(V3D_PROJ_TEST_CLIP_WIN | V3D_PROJ_TEST_CLIP_BB);
-
-  if (ED_view3d_project_float_object(data->vc.region, v0co_clip, screen_co_a, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
-    return;
-  }
-  if (ED_view3d_project_float_object(data->vc.region, v1co_clip, screen_co_b, clip_flag_nowin) !=
-      V3D_PROJ_RET_OK) {
+  float screen_co_a[2], screen_co_b[2];
+  if (!mesh_foreachScreenEdge_shared_project_and_test(data->vc.region,
+                                                      v0co_clip,
+                                                      v1co_clip,
+                                                      data->clip_flag,
+                                                      &data->win_rect,
+                                                      data->content_planes,
+                                                      data->content_planes_len,
+                                                      screen_co_a,
+                                                      screen_co_b)) {
     return;
   }
 
-  if (data->clip_flag & V3D_PROJ_TEST_CLIP_WIN) {
-    if (!BLI_rctf_isect_segment(&data->win_rect, screen_co_a, screen_co_b)) {
-      return;
-    }
-  }
-
   data->func(data->userData, eed, screen_co_a, screen_co_b, index);
 }
 
@@ -341,6 +460,14 @@ void mesh_foreachScreenEdge_clip_bb_segment(ViewContext *vc,
   data.userData = userData;
   data.clip_flag = clip_flag;
 
+  if (clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) {
+    data.content_planes_len = content_planes_from_clip_flag(
+        vc->region, vc->obedit, clip_flag, data.content_planes);
+  }
+  else {
+    data.content_planes_len = 0;
+  }
+
   BM_mesh_elem_table_ensure(vc->em->bm, BM_EDGE);
 
   if ((clip_flag & V3D_PROJ_TEST_CLIP_BB) && (vc->rv3d->clipbb != NULL)) {
@@ -380,6 +507,7 @@ void mesh_foreachScreenFace(
     void *userData,
     const eV3DProjTest clip_flag)
 {
+  BLI_assert((clip_flag & V3D_PROJ_TEST_CLIP_CONTENT) == 0);
   foreachScreenFace_userData data;
 
   Mesh *me = editbmesh_get_eval_cage_from_orig(
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 6a7ee46f719..5bdeb2cb35f 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -876,11 +876,16 @@ static bool do_lasso_select_mesh(ViewContext *vc,
 
     const eV3DProjTest clip_flag = V3D_PROJ_TEST_CLIP_NEAR |
                                    (use_zbuf ? 0 : V3D_PROJ_TEST_CLIP_BB);
+    /* Fully inside. */
     mesh_foreachScreenEdge_clip_bb_segment(
         vc, do_lasso_select_mesh__doSelectEdge_pass0, &data_for_edge, clip_flag);
     if (data.is_done == false) {
-      mesh_foreachScreenEdge_clip_bb_segment(
-          vc, do_lasso_select_mesh__doSelectEdge_pass1, &data_for_edge, clip_flag);
+      /* Fall back to partially inside.
+       * Clip content to account for edges partially behind the view. */
+      mesh_foreachScreenEdge_clip_bb_segment(vc,
+                                             do_lasso_select_mesh__doSelectEdge_pass1,
+                                             &data_for_edge,
+                                             clip_flag | V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT);
     }
   }
 
@@ -3071,6 +3076,9 @@ struct BoxSelectUserData_ForMeshEdge {
   struct EditSelectBuf_Cache *esel;
   uint backbuf_offset;
 };
+/**
+ * Pass 0 operates on edges when fully inside.
+ */
 static void do_mesh_box_select__doSelectEdge_pass0(
     void *userData, BMEdge *eed, const float screen_co_a[2], const float screen_co_b[2], int index)
 {
@@ -3092,6 +3100,9 @@ static void do_mesh_box_select__doSelectEdge_pass0(
     data->is_changed = true;
   }
 }
+/**
+ * Pass 1 operates on edges when partially inside.
+ */
 static void do_mesh_box_select__doSelectEdge_pass1(
     void *userData, BMEdge *eed, const float screen_co_a[2], const float screen_co_b[2], int index)
 {
@@ -3181,11 +3192,16 @@ static bool do_mesh_box_select(ViewContext *vc,
 
     const eV3DProjTest clip_flag = V3D_PROJ_TEST_CLIP_NEAR |
                                    (use_zbuf ? 0 : V3D_PROJ_TEST_CLIP_BB);
+    /* Fully inside. */
     mesh_foreachScreenEdge_clip_bb_segment(
         vc, do_mesh_box_select__doSelectEdge_pass0, &cb_data, clip_flag);
     if (data.is_done == false) {
-      mesh_foreachScreenEdge_clip_bb_segment(
-          vc, do_mesh_box_select__doSelectEdge_pass1, &cb_data, clip_flag);
+      /* Fall back to partially inside.
+       * Clip content to account for edges partially behind the view. */
+      mesh_foreachScreenEdge_clip_bb_segment(vc,
+                                             do_mesh_box_select__doSelectEdge_pass1,
+                                             &cb_data,
+                                             clip_flag | V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT);
     }
   }
 
@@ -3774,7 +3790,10 @@ static bool mesh_circle_select(ViewContext *vc,
     }
     else {
       mesh_foreachScreenEdge_clip_bb_segment(
-          vc, mesh_circle_doSelectEdge, &data, V3D_PROJ_TEST_CLIP_NEAR | V3D_PROJ_TEST_CLIP_BB);
+          vc,
+          mesh_circle_doSelectEdge,
+          &data,
+          (V3D_PROJ_TEST_CLIP_NEAR | V3D_PROJ_TEST_CLIP_BB | V3D_PROJ_TEST_CLIP_CONTENT_DEFAULT));
     }
   }