Fix dist_signed_squared_to_corner_v3v3v3 with concave corners

2 files changed, 23 insertions, 9 deletions

diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index f9cce27b1cb..9023cdd17bf 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -107,7 +107,7 @@ float         dist_to_line_v3(const float p[3], const float l1[3], const float l
 float dist_signed_squared_to_corner_v3v3v3(
         const float p[3],
         const float v1[3], const float v2[3], const float v3[3],
-        const float axis_fallback[3]);
+        const float axis_ref[3]);
 float closest_to_line_v3(float r[3], const float p[3], const float l1[3], const float l2[3]);
 float closest_to_line_v2(float r[2], const float p[2], const float l1[2], const float l2[2]);
 void closest_to_line_segment_v3(float r_close[3], const float p[3], const float l1[3], const float l2[3]);
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index 92fa52afb80..eee78297106 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -481,7 +481,7 @@ float dist_to_line_v3(const float v1[3], const float l1[3], const float l2[3])
  * Check if \a p is inside the 2x planes defined by ``(v1, v2, v3)``
  * where the 3x points define 2x planes.
  *
- * \param axis_fallback used when v1,v2,v3 form a line.
+ * \param axis_ref used when v1,v2,v3 form a line and to check if the corner is concave/convex.
  *
  * \note the distance from \a v1 & \a v3 to \a v2 doesnt matter
  * (it just defines the planes).
@@ -503,23 +503,30 @@ float dist_to_line_v3(const float v1[3], const float l1[3], const float l2[3])
 float dist_signed_squared_to_corner_v3v3v3(
         const float p[3],
         const float v1[3], const float v2[3], const float v3[3],
-        const float axis_fallback[3])
+        const float axis_ref[3])
 {
 	float dir_a[3], dir_b[3];
 	float plane_a[4], plane_b[4];
+	float dist_a, dist_b;
 	float axis[3];
+	bool flip = false;
 
 	sub_v3_v3v3(dir_a, v1, v2);
 	sub_v3_v3v3(dir_b, v3, v2);
 
 	cross_v3_v3v3(axis, dir_a, dir_b);
 
-	if ((len_squared_v3(axis) < FLT_EPSILON) && axis_fallback) {
-		copy_v3_v3(axis, axis_fallback);
+	if ((len_squared_v3(axis) < FLT_EPSILON)) {
+		copy_v3_v3(axis, axis_ref);
+	}
+	else if (dot_v3v3(axis, axis_ref) < 0.0f) {
+		/* concave */
+		flip = true;
+		negate_v3(axis);
 	}
 
-	cross_v3_v3v3(plane_a, axis, dir_a);
-	cross_v3_v3v3(plane_b, dir_b, axis);
+	cross_v3_v3v3(plane_a, dir_a, axis);
+	cross_v3_v3v3(plane_b, axis, dir_b);
 
 #if 0
 	plane_from_point_normal_v3(plane_a, center, l1);
@@ -530,8 +537,15 @@ float dist_signed_squared_to_corner_v3v3v3(
 	plane_b[3] = -dot_v3v3(plane_b, v2);
 #endif
 
-	return min_ff(dist_signed_squared_to_plane_v3(p, plane_a),
-	              dist_signed_squared_to_plane_v3(p, plane_b));
+	dist_a = dist_signed_squared_to_plane_v3(p, plane_a);
+	dist_b = dist_signed_squared_to_plane_v3(p, plane_b);
+
+	if (flip) {
+		return min_ff(dist_a, dist_b);
+	}
+	else {
+		return max_ff(dist_a, dist_b);
+	}
 }
 
 /* Adapted from "Real-Time Collision Detection" by Christer Ericson,