fix [#32315] Circle select unreliable when in vertex+edge select mode

dist_squared_to_line_segment_v2() was returning the sqrt'd value in some cases.
also use int's for edge_inside_circle() rather then shorts since it was doing int/float/short conversions and we're now using int's for screen vars in more places.

2 files changed, 21 insertions, 19 deletions

diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index 34892fb3c27..7f40024a056 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -268,7 +268,7 @@ int view3d_get_view_aligned_coordinate(struct ViewContext *vc, float fp[3], cons
 void view3d_get_transformation(const struct ARegion *ar, struct RegionView3D *rv3d, struct Object *ob, struct bglMats *mats);
 
 /* XXX should move to BLI_math */
-int edge_inside_circle(short centx, short centy, short rad, short x1, short y1, short x2, short y2);
+int edge_inside_circle(int centx, int centy, int rad, int x1, int y1, int x2, int y2);
 
 /* get 3d region from context, also if mouse is in header or toolbar */
 struct RegionView3D *ED_view3d_context_rv3d(struct bContext *C);
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 1a91b0aa085..69fa42b0abb 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -1523,26 +1523,28 @@ typedef struct BoxSelectUserData {
 	int select, pass, done;
 } BoxSelectUserData;
 
-int edge_inside_circle(short centx, short centy, short rad, short x1, short y1, short x2, short y2)
+int edge_inside_circle(int centx, int centy, int rad, int x1, int y1, int x2, int y2)
 {
 	int radsq = rad * rad;
-	float v1[2], v2[2], v3[2];
-	
+
 	/* check points in circle itself */
-	if ( (x1 - centx) * (x1 - centx) + (y1 - centy) * (y1 - centy) <= radsq) return 1;
-	if ( (x2 - centx) * (x2 - centx) + (y2 - centy) * (y2 - centy) <= radsq) return 1;
-	
-	/* pointdistline */
-	v3[0] = centx;
-	v3[1] = centy;
-	v1[0] = x1;
-	v1[1] = y1;
-	v2[0] = x2;
-	v2[1] = y2;
-	
-	if (dist_to_line_segment_v2(v3, v1, v2) < (float)rad) return 1;
-	
-	return 0;
+	if ( (x1 - centx) * (x1 - centx) + (y1 - centy) * (y1 - centy) <= radsq) {
+		return TRUE;
+	}
+	else if ( (x2 - centx) * (x2 - centx) + (y2 - centy) * (y2 - centy) <= radsq) {
+		return TRUE;
+	}
+	else {
+		const float cent[2] = {centx, centy};
+		const float v1[2] = {x1, y1};
+		const float v2[2] = {x2, y2};
+		/* pointdistline */
+		if (dist_squared_to_line_segment_v2(cent, v1, v2) < (float)radsq) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
 }
 
 static void do_nurbs_box_select__doSelect(void *userData, Nurb *UNUSED(nu), BPoint *bp, BezTriple *bezt, int beztindex, int x, int y)
@@ -2192,7 +2194,7 @@ static void mesh_circle_doSelectEdge(void *userData, BMEdge *eed, int x0, int y0
 {
 	CircleSelectUserData *data = userData;
 
-	if (edge_inside_circle(data->mval[0], data->mval[1], (short) data->radius, x0, y0, x1, y1)) {
+	if (edge_inside_circle(data->mval[0], data->mval[1], (int)data->radius, x0, y0, x1, y1)) {
 		BM_edge_select_set(data->vc->em->bm, eed, data->select);
 	}
 }