fix a glitch where overlapping feathering could give random bad pixels, this was caused by feather edge going in a different direction to the mask edge - creating bowtie quads.

1 files changed, 7 insertions, 2 deletions

diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index b908a32bf4c..f3b047130f2 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -1991,8 +1991,13 @@ void barycentric_weights_v2(const float v1[2], const float v2[2], const float v3
 void barycentric_weights_v2_quad(const float v1[2], const float v2[2], const float v3[2], const float v4[2],
                                  const float co[2], float w[4])
 {
-#define MEAN_VALUE_HALF_TAN_V2(_area, i1, i2) ((_area = cross_v2v2(dirs[i1], dirs[i2])) != 0.0f ? \
-	                                           (((lens[i1] * lens[i2]) - dot_v2v2(dirs[i1], dirs[i2])) / _area) : 0.0f)
+	/* note: fabsf() here is not needed for convex quads (and not used in interp_weights_poly_v2).
+	 *       but in the case of concave/bowtie quads for the mask rasterizer it gives unreliable results
+	 *       without adding absf(). If this becomes an issue for more general useage we could have
+	 *       this optional or use a different function - Campbell */
+#define MEAN_VALUE_HALF_TAN_V2(_area, i1, i2) \
+	        ((_area = cross_v2v2(dirs[i1], dirs[i2])) != 0.0f ? \
+	         fabsf(((lens[i1] * lens[i2]) - dot_v2v2(dirs[i1], dirs[i2])) / _area) : 0.0f)
 
 	float wtot, area;