fix for lasso selection (in non-zbuf mode) when the line intersected its self.

isect_point_poly_v2() - add argument to check overlapping areas.

3 files changed, 25 insertions, 7 deletions

diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index d3f6c7ef14b..c7ef410a606 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -155,8 +155,8 @@ bool isect_ray_tri_epsilon_v3(const float p1[3], const float d[3],
                               const float v0[3], const float v1[3], const float v2[3], float *r_lambda, float r_uv[2], const float epsilon);
 
 /* point in polygon */
-bool isect_point_poly_v2(const float pt[2], const float verts[][2], const unsigned int nr);
-bool isect_point_poly_v2_int(const int pt[2], const int verts[][2], const unsigned int nr);
+bool isect_point_poly_v2(const float pt[2], const float verts[][2], const unsigned int nr, const bool use_overlap);
+bool isect_point_poly_v2_int(const int pt[2], const int verts[][2], const unsigned int nr, const bool use_overlap);
 
 int isect_point_quad_v2(const float p[2], const float a[2], const float b[2], const float c[2], const float d[2]);
 
diff --git a/source/blender/blenlib/intern/lasso.c b/source/blender/blenlib/intern/lasso.c
index 8e22d6f1182..e89f7fd795b 100644
--- a/source/blender/blenlib/intern/lasso.c
+++ b/source/blender/blenlib/intern/lasso.c
@@ -63,7 +63,7 @@ bool BLI_lasso_is_point_inside(const int mcords[][2], const unsigned int moves,
 	}
 	else {
 		int pt[2] = {sx, sy};
-		return isect_point_poly_v2_int(pt, mcords, moves);
+		return isect_point_poly_v2_int(pt, mcords, moves, true);
 	}
 }
 
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index 65cfac86021..e45714f33da 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -712,7 +712,8 @@ int isect_line_sphere_v2(const float l1[2], const float l2[2],
 }
 
 /* point in polygon (keep float and int versions in sync) */
-bool isect_point_poly_v2(const float pt[2], const float verts[][2], const unsigned int nr)
+bool isect_point_poly_v2(const float pt[2], const float verts[][2], const unsigned int nr,
+                         const bool use_overlap)
 {
 	/* we do the angle rule, define that all added angles should be about zero or (2 * PI) */
 	float angletot = 0.0;
@@ -749,9 +750,18 @@ bool isect_point_poly_v2(const float pt[2], const float verts[][2], const unsign
 		p1 = p2;
 	}
 
-	return (fabsf(angletot) > 4.0f);
+	angletot = fabsf(angletot);
+	if (use_overlap) {
+		const int nested = floorf((angletot / (float)(M_PI * 2.0)) + 0.00001f);
+		angletot -= nested * (float)(M_PI * 2.0);
+		return (angletot > 4.0f) != (nested % 2);
+	}
+	else {
+		return (angletot > 4.0f);
+	}
 }
-bool isect_point_poly_v2_int(const int pt[2], const int verts[][2], const unsigned int nr)
+bool isect_point_poly_v2_int(const int pt[2], const int verts[][2], const unsigned int nr,
+                             const bool use_overlap)
 {
 	/* we do the angle rule, define that all added angles should be about zero or (2 * PI) */
 	float angletot = 0.0;
@@ -788,7 +798,15 @@ bool isect_point_poly_v2_int(const int pt[2], const int verts[][2], const unsign
 		p1 = p2;
 	}
 
-	return (fabsf(angletot) > 4.0f);
+	angletot = fabsf(angletot);
+	if (use_overlap) {
+		const int nested = floorf((angletot / (float)(M_PI * 2.0)) + 0.00001f);
+		angletot -= nested * (float)(M_PI * 2.0);
+		return (angletot > 4.0f) != (nested % 2);
+	}
+	else {
+		return (angletot > 4.0f);
+	}
 }
 
 /* point in tri */