Fix #30638 and part of #30646.

Problem was that area calculation of polygons was done relative to the xy plane, and with a very obscure (to me at least) algorithm. That meant that vertical ngons would get 0 area. 

Commented initial code in case this is a strange optimization case that someone wants to use and used a cleaner algorithm: first project vertices to the ngon plane, defined by the normal of the ngon and the center (mean) of the ngon vertices. This will only be exact for convex and mostly planar ngons, still it is much better than the previous code.

Also fixed memory leak when stretch display was on.

3 files changed, 21 insertions, 0 deletions

diff --git a/source/blender/blenlib/BLI_math_vector.h b/source/blender/blenlib/BLI_math_vector.h
index b8b53f1ba7c..b4e1a71d45e 100644
--- a/source/blender/blenlib/BLI_math_vector.h
+++ b/source/blender/blenlib/BLI_math_vector.h
@@ -129,6 +129,7 @@ MINLINE void star_m3_v3(float rmat[3][3],float a[3]);
 /*********************************** Length **********************************/
 
 MINLINE float len_squared_v2(const float v[2]);
+MINLINE float len_squared_v3(const float v[3]);
 MINLINE float len_v2(const float a[2]);
 MINLINE float len_v2v2(const float a[2], const float b[2]);
 MINLINE float len_squared_v2v2(const float a[2], const float b[2]);
@@ -190,6 +191,7 @@ void angle_poly_v3(float* angles, const float* verts[3], int len);
 
 void project_v2_v2v2(float c[2], const float v1[2], const float v2[2]);
 void project_v3_v3v3(float r[3], const float p[3], const float n[3]);
+void project_v3_plane(float v[3], const float n[3], const float p[3]);
 void reflect_v3_v3v3(float r[3], const float v[3], const float n[3]);
 void ortho_basis_v3v3_v3(float r1[3], float r2[3], const float a[3]);
 void bisect_v3_v3v3v3(float r[3], const float a[3], const float b[3], const float c[3]);
diff --git a/source/blender/blenlib/intern/math_vector.c b/source/blender/blenlib/intern/math_vector.c
index ce387291dff..e7c96ce3dc3 100644
--- a/source/blender/blenlib/intern/math_vector.c
+++ b/source/blender/blenlib/intern/math_vector.c
@@ -286,6 +286,20 @@ void project_v3_v3v3(float c[3], const float v1[3], const float v2[3])
 	c[2] = mul * v2[2];
 }
 
+/* project a vector on a plane defined by normal and a plane point p */
+void project_v3_plane(float v[3], const float n[3], const float p[3])
+{
+	float vector[3];
+	float mul;
+
+	sub_v3_v3v3(vector, v, p);
+	mul = dot_v3v3(vector, n)/len_squared_v3(n);
+
+	mul_v3_v3fl(vector, n, mul);
+
+	sub_v3_v3(v, vector);
+}
+
 /* Returns a vector bisecting the angle at v2 formed by v1, v2 and v3 */
 void bisect_v3_v3v3v3(float out[3], const float v1[3], const float v2[3], const float v3[3])
 {
diff --git a/source/blender/blenlib/intern/math_vector_inline.c b/source/blender/blenlib/intern/math_vector_inline.c
index 6d217b48d37..62e582c89c4 100644
--- a/source/blender/blenlib/intern/math_vector_inline.c
+++ b/source/blender/blenlib/intern/math_vector_inline.c
@@ -498,6 +498,11 @@ MINLINE float len_squared_v2(const float v[2])
 	return v[0] * v[0] + v[1] * v[1];
 }
 
+MINLINE float len_squared_v3(const float v[3])
+{
+	return v[0] * v[0] + v[1] * v[1] + v[2] * v[2];
+}
+
 MINLINE float len_v2(const float v[2])
 {
 	return (float)sqrtf(v[0] * v[0] + v[1] * v[1]);