BMesh: BM_loop/edge_point_side_of_loop_test

change behavior to use a negative number when outside,
and return the signed, squared distance.

1 files changed, 9 insertions, 8 deletions

diff --git a/source/blender/bmesh/intern/bmesh_queries.c b/source/blender/bmesh/intern/bmesh_queries.c
index c3fa1e6902f..781f6eb73ad 100644
--- a/source/blender/bmesh/intern/bmesh_queries.c
+++ b/source/blender/bmesh/intern/bmesh_queries.c
@@ -273,30 +273,31 @@ static float bm_face_calc_split_dot(BMLoop *l_a, BMLoop *l_b)
  * Check if a point is inside the corner defined by a loop
  * (within the 2 planes defined by the loops corner & face normal).
  *
- * \return less than 0.0 when inside.
+ * \return signed, squared distance to the loops planes, less than 0.0 when outside.
  */
 float BM_loop_point_side_of_loop_test(const BMLoop *l, const float co[3])
 {
 	const float *axis = l->f->no;
-	return (angle_signed_on_axis_v3v3v3_v3(l->prev->v->co, l->v->co, co,             axis) -
-	        angle_signed_on_axis_v3v3v3_v3(l->prev->v->co, l->v->co, l->next->v->co, axis));
+	return dist_signed_squared_to_corner_v3v3v3(co, l->prev->v->co, l->v->co, l->next->v->co, axis);
 }
 
 /**
  * Check if a point is inside the edge defined by a loop
  * (within the plane defined by the loops edge & face normal).
  *
- * \return less than 0.0 when inside.
+ * \return signed, squared distablce to the edge plane, less than 0.0 when outside.
  */
 float BM_loop_point_side_of_edge_test(const BMLoop *l, const float co[3])
 {
 	const float *axis = l->f->no;
 	float dir[3];
-	float plane[3];
-	sub_v3_v3v3(dir, l->v->co, l->next->v->co);
+	float plane[4];
+
+	sub_v3_v3v3(dir, l->next->v->co, l->v->co);
 	cross_v3_v3v3(plane, axis, dir);
-	return (dot_v3v3(plane, co) -
-	        dot_v3v3(plane, l->v->co));
+
+	plane[3] = -dot_v3v3(plane, l->v->co);
+	return dist_signed_squared_to_plane_v3(co, plane);
 }
 
 /**