fix for mask feather intersection checks not working right for non-filled, feathered masks.

now do intersection checks for both sides of the feather.

3 files changed, 46 insertions, 10 deletions

diff --git a/source/blender/blenkernel/BKE_mask.h b/source/blender/blenkernel/BKE_mask.h
index 1836a88555b..a56f99c011c 100644
--- a/source/blender/blenkernel/BKE_mask.h
+++ b/source/blender/blenkernel/BKE_mask.h
@@ -74,9 +74,10 @@ float (*BKE_mask_spline_feather_differentiated_points(struct MaskSpline *spline,
 
 float (*BKE_mask_spline_differentiate_with_resolution_ex(struct MaskSpline *spline, int *tot_diff_point,
                                                          const unsigned int resol))[2];
+void    BKE_mask_spline_feather_collapse_inner_loops(struct MaskSpline *spline, float (*feather_points)[2], const int tot_feather_point);
 float (*BKE_mask_spline_differentiate_with_resolution(struct MaskSpline *spline, int width, int height, int *tot_diff_point))[2];
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(struct MaskSpline *spline, int *tot_feather_point,
-                                                                         const unsigned int resol))[2];
+                                                                         const unsigned int resol, const int do_feather_isect))[2];
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution(struct MaskSpline *spline, int width, int height, int *tot_feather_point))[2];
 
 float (*BKE_mask_spline_feather_points(struct MaskSpline *spline, int *tot_feather_point))[2];
diff --git a/source/blender/blenkernel/intern/mask.c b/source/blender/blenkernel/intern/mask.c
index 3c46e7bcd47..d951f430dc2 100644
--- a/source/blender/blenkernel/intern/mask.c
+++ b/source/blender/blenkernel/intern/mask.c
@@ -559,7 +559,7 @@ static void feather_bucket_get_diagonal(FeatherEdgesBucket *buckets, int start_b
 	*diagonal_bucket_b_r = &buckets[diagonal_bucket_b_index];
 }
 
-static void spline_feather_collapse_inner_loops(MaskSpline *spline, float (*feather_points)[2], int tot_feather_point)
+void BKE_mask_spline_feather_collapse_inner_loops(MaskSpline *spline, float (*feather_points)[2], const int tot_feather_point)
 {
 #define BUCKET_INDEX(co) \
 	feather_bucket_index_from_coord(co, min, bucket_scale, buckets_per_side)
@@ -721,7 +721,8 @@ static void spline_feather_collapse_inner_loops(MaskSpline *spline, float (*feat
  */
 float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(MaskSpline *spline,
                                                                          int *tot_feather_point,
-                                                                         const unsigned int resol
+                                                                         const unsigned int resol,
+                                                                         const int do_feather_isect
                                                                          ))[2]
 {
 	MaskSplinePoint *points_array = BKE_mask_spline_point_array(spline);
@@ -783,8 +784,9 @@ float (*BKE_mask_spline_feather_differentiated_points_with_resolution_ex(MaskSpl
 
 	*tot_feather_point = tot;
 
-	if (spline->flag & MASK_SPLINE_NOINTERSECT)
-		spline_feather_collapse_inner_loops(spline, feather, tot);
+	if ((spline->flag & MASK_SPLINE_NOINTERSECT) && do_feather_isect) {
+		BKE_mask_spline_feather_collapse_inner_loops(spline, feather, tot);
+	}
 
 	return feather;
 }
@@ -794,7 +796,7 @@ float (*BKE_mask_spline_feather_differentiated_points_with_resolution(MaskSpline
 {
 	unsigned int resol = BKE_mask_spline_feather_resolution(spline, width, height);
 
-	return BKE_mask_spline_feather_differentiated_points_with_resolution_ex(spline, tot_feather_point, resol);
+	return BKE_mask_spline_feather_differentiated_points_with_resolution_ex(spline, tot_feather_point, resol, TRUE);
 }
 
 float (*BKE_mask_spline_feather_differentiated_points(MaskSpline *spline, int *tot_feather_point))[2]
diff --git a/source/blender/blenkernel/intern/mask_rasterize.c b/source/blender/blenkernel/intern/mask_rasterize.c
index 6f7cf43be69..d39be3b8ed6 100644
--- a/source/blender/blenkernel/intern/mask_rasterize.c
+++ b/source/blender/blenkernel/intern/mask_rasterize.c
@@ -575,6 +575,7 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 			int tot_diff_point;
 
 			float (*diff_feather_points)[2];
+			float (*diff_feather_points_flip)[2];
 			int tot_diff_feather_points;
 
 			const unsigned int resol_a = BKE_mask_spline_resolution(spline, width, height) / 4;
@@ -586,7 +587,7 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 
 			if (do_feather) {
 				diff_feather_points = BKE_mask_spline_feather_differentiated_points_with_resolution_ex(
-				                          spline, &tot_diff_feather_points, resol);
+				                          spline, &tot_diff_feather_points, resol, FALSE);
 				BLI_assert(diff_feather_points);
 			}
 			else {
@@ -649,6 +650,11 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 				}
 
 				if (is_fill) {
+					/* applt intersections depending on fill settings */
+					if (spline->flag & MASK_SPLINE_NOINTERSECT) {
+						BKE_mask_spline_feather_collapse_inner_loops(spline, diff_feather_points, tot_diff_feather_points);
+					}
+
 					copy_v2_v2(co, diff_points[0]);
 					sf_vert_prev = BLI_scanfill_vert_add(&sf_ctx, co);
 					sf_vert_prev->tmp.u = sf_vert_tot;
@@ -710,11 +716,27 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 					/* unfilled spline */
 					if (diff_feather_points) {
 
-						float co_diff[3];
+						float co_diff[2];
 
 						float co_feather[3];
 						co_feather[2] = 1.0f;
 
+						if (spline->flag & MASK_SPLINE_NOINTERSECT) {
+							diff_feather_points_flip = MEM_mallocN(sizeof(float) * 2 * tot_diff_feather_points, "diff_feather_points_flip");
+
+							for (j = 0; j < tot_diff_point; j++) {
+								sub_v2_v2v2(co_diff, diff_points[j], diff_feather_points[j]);
+								add_v2_v2v2(diff_feather_points_flip[j], diff_points[j], co_diff);
+							}
+
+							BKE_mask_spline_feather_collapse_inner_loops(spline, diff_feather_points,      tot_diff_feather_points);
+							BKE_mask_spline_feather_collapse_inner_loops(spline, diff_feather_points_flip, tot_diff_feather_points);
+						}
+						else {
+							diff_feather_points_flip = NULL;
+						}
+
+
 						open_spline_ranges[open_spline_index].vertex_offset = sf_vert_tot;
 						open_spline_ranges[open_spline_index].vertex_total = tot_diff_point;
 
@@ -738,8 +760,14 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 
 
 							/* feather vert B */
-							sub_v2_v2v2(co_diff, co, co_feather);
-							add_v2_v2v2(co_feather, co, co_diff);
+							if (diff_feather_points_flip) {
+								copy_v2_v2(co_feather, diff_feather_points_flip[j]);
+							}
+							else {
+								sub_v2_v2v2(co_diff, co, co_feather);
+								add_v2_v2v2(co_feather, co, co_diff);
+							}
+
 							sf_vert = BLI_scanfill_vert_add(&sf_ctx, co_feather);
 							sf_vert->tmp.u = sf_vert_tot;
 							sf_vert->keyindex = SF_KEYINDEX_TEMP_ID;
@@ -752,6 +780,11 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 							tot_feather_quads -= 2;
 						}
 
+						if (diff_feather_points_flip) {
+							MEM_freeN(diff_feather_points_flip);
+							diff_feather_points_flip = NULL;
+						}
+
 						/* cap ends */
 
 						/* dummy init value */