fix [#30848] Edge Selection fails when the following Conditions met:

summery: when both verts are outside the viewport

this is in fact a very old annoyance but good to resolve.
fix by doing segment/rectangle intersection between the edge and the viewport so it works when the both verts are outside the view.

3 files changed, 63 insertions, 11 deletions

diff --git a/source/blender/blenlib/BLI_rect.h b/source/blender/blenlib/BLI_rect.h
index d1209524c15..7ce19ce8583 100644
--- a/source/blender/blenlib/BLI_rect.h
+++ b/source/blender/blenlib/BLI_rect.h
@@ -58,6 +58,8 @@ void BLI_resize_rcti(struct rcti *rect, int x, int y);
 void BLI_resize_rctf(struct rctf *rect, float x, float y);
 int  BLI_in_rcti(struct rcti *rect, int x, int y);
 int  BLI_in_rctf(struct rctf *rect, float x, float y);
+int  BLI_segment_in_rcti(struct rcti *rect, int s1[2], int s2[2]);
+// int  BLI_segment_in_rctf(struct rcti *rect, int s1[2], int s2[2]); // NOT NEEDED YET
 int  BLI_isect_rctf(struct rctf *src1, struct rctf *src2, struct rctf *dest);
 int  BLI_isect_rcti(struct rcti *src1, struct rcti *src2, struct rcti *dest);
 void BLI_union_rctf(struct rctf *rcta, struct rctf *rctb);
diff --git a/source/blender/blenlib/intern/rct.c b/source/blender/blenlib/intern/rct.c
index d3bb5a1b17f..81a940e4613 100644
--- a/source/blender/blenlib/intern/rct.c
+++ b/source/blender/blenlib/intern/rct.c
@@ -66,6 +66,55 @@ int BLI_in_rctf(rctf *rect, float x, float y)
 	return 1;
 }
 
+/* based closely on 'isect_line_line_v2_int', but in modified so corner cases are treated as intersections */
+static int isect_segments(const int v1[2], const int v2[2], const int v3[2], const int v4[2])
+{
+	const double div = (double)((v2[0] - v1[0]) * (v4[1] - v3[1]) - (v2[1] - v1[1]) * (v4[0] - v3[0]));
+	if (div == 0.0f) {
+		return 1; /* co-linear */
+	}
+	else {
+		const double labda = (double)((v1[1] - v3[1]) * (v4[0] - v3[0]) - (v1[0] - v3[0]) * (v4[1] - v3[1])) / div;
+		const double mu    = (double)((v1[1] - v3[1]) * (v2[0] - v1[0]) - (v1[0] - v3[0]) * (v2[1] - v1[1])) / div;
+		return (labda >= 0.0f && labda <= 1.0f && mu >= 0.0f && mu <= 1.0f);
+	}
+}
+
+int BLI_segment_in_rcti(rcti *rect, int s1[2], int s2[2])
+{
+	/* first do outside-bounds check for both points of the segment */
+	if (s1[0] < rect->xmin && s2[0] < rect->xmin) return 0;
+	if (s1[0] > rect->xmax && s2[0] > rect->xmax) return 0;
+	if (s1[1] < rect->ymin && s2[1] < rect->ymin) return 0;
+	if (s1[1] > rect->ymax && s2[1] > rect->ymax) return 0;
+
+	/* if either points intersect then we definetly intersect */
+	if (BLI_in_rcti(rect, s1[0], s1[1]) || BLI_in_rcti(rect, s2[0], s2[1])) {
+		return 1;
+	}
+	else {
+		/* both points are outside but may insersect the rect */
+		int tvec1[2];
+		int tvec2[2];
+		/* diagonal: [/] */
+		tvec1[0] = rect->xmin; tvec1[1] = rect->ymin;
+		tvec2[0] = rect->xmin; tvec2[1] = rect->ymax;
+		if (isect_segments(s1, s2, tvec1, tvec2)) {
+			return 1;
+		}
+
+		/* diagonal: [\] */
+		tvec1[0] = rect->xmin; tvec1[1] = rect->ymax;
+		tvec2[0] = rect->xmax; tvec2[1] = rect->ymin;
+		if (isect_segments(s1, s2, tvec1, tvec2)) {
+			return 1;
+		}
+
+		/* no intersection */
+		return 0;
+	}
+}
+
 void BLI_union_rctf(rctf *rct1, rctf *rct2)
 {
 	if (rct1->xmin > rct2->xmin) rct1->xmin = rct2->xmin;
diff --git a/source/blender/editors/space_view3d/drawobject.c b/source/blender/editors/space_view3d/drawobject.c
index 177a5d3d850..97e52a21e25 100644
--- a/source/blender/editors/space_view3d/drawobject.c
+++ b/source/blender/editors/space_view3d/drawobject.c
@@ -127,6 +127,7 @@ typedef struct foreachScreenEdge_userData {
 	void (*func)(void *userData, BMEdge *eed, int x0, int y0, int x1, int y1, int index);
 	void *userData;
 	ViewContext vc;
+	rcti win_rect; /* copy of: vc.ar->winx/winy, use for faster tests, minx/y will always be 0 */
 	eV3DClipTest clipVerts;
 } foreachScreenEdge_userData;
 
@@ -2099,14 +2100,7 @@ static void drawSelectedVertices(DerivedMesh *dm, Mesh *me)
 	dm->foreachMappedVert(dm, drawSelectedVertices__mapFunc, me->mvert);
 	glEnd();
 }
-static int is_co_in_region(ARegion *ar, const short co[2])
-{
-	return ((co[0] != IS_CLIPPED) && /* may be the only initialized value, check first */
-	        (co[0] >= 0)          &&
-	        (co[0] <  ar->winx)   &&
-	        (co[1] >= 0)          &&
-	        (co[1] <  ar->winy));
-}
+
 static void mesh_foreachScreenEdge__mapFunc(void *userData, int index, const float v0co[3], const float v1co[3])
 {
 	foreachScreenEdge_userData *data = userData;
@@ -2129,9 +2123,10 @@ static void mesh_foreachScreenEdge__mapFunc(void *userData, int index, const flo
 			project_short_noclip(data->vc.ar, v2_co, s[1]);
 
 			if (data->clipVerts == V3D_CLIP_TEST_REGION) {
-				if (!is_co_in_region(data->vc.ar, s[0]) &&
-				    !is_co_in_region(data->vc.ar, s[1]))
-				{
+				/* make an int copy */
+				int s_int[2][2] = {{s[0][0], s[0][1]},
+				                   {s[1][0], s[1][1]}};
+				if (!BLI_segment_in_rcti(&data->win_rect, s_int[0], s_int[1])) {
 					return;
 				}
 			}
@@ -2150,6 +2145,12 @@ void mesh_foreachScreenEdge(
 	DerivedMesh *dm = editbmesh_get_derived_cage(vc->scene, vc->obedit, vc->em, CD_MASK_BAREMESH);
 
 	data.vc = *vc;
+
+	data.win_rect.xmin = 0;
+	data.win_rect.ymin = 0;
+	data.win_rect.xmax = vc->ar->winx;
+	data.win_rect.ymax = vc->ar->winy;
+
 	data.func = func;
 	data.userData = userData;
 	data.clipVerts = clipVerts;