Mask: add option to detect self intersections

2 files changed, 44 insertions, 1 deletions

diff --git a/source/blender/blenkernel/intern/displist.c b/source/blender/blenkernel/intern/displist.c
index 517c8329567..07890e84d70 100644
--- a/source/blender/blenkernel/intern/displist.c
+++ b/source/blender/blenkernel/intern/displist.c
@@ -459,6 +459,7 @@ void BKE_displist_fill(ListBase *dispbase, ListBase *to, const float normal_proj
 	float *f1;
 	int colnr = 0, charidx = 0, cont = 1, tot, a, *index, nextcol = 0;
 	int totvert;
+	const int scanfill_flag = BLI_SCANFILL_CALC_REMOVE_DOUBLES | BLI_SCANFILL_CALC_POLYS | BLI_SCANFILL_CALC_HOLES;
 
 	if (dispbase == NULL)
 		return;
@@ -519,7 +520,7 @@ void BKE_displist_fill(ListBase *dispbase, ListBase *to, const float normal_proj
 
 		/* XXX (obedit && obedit->actcol) ? (obedit->actcol-1) : 0)) { */
 		if (totvert && (tot = BLI_scanfill_calc_ex(&sf_ctx,
-		                                           BLI_SCANFILL_CALC_REMOVE_DOUBLES | BLI_SCANFILL_CALC_HOLES,
+		                                           scanfill_flag,
 		                                           normal_proj)))
 		{
 			if (tot) {
diff --git a/source/blender/blenkernel/intern/mask_rasterize.c b/source/blender/blenkernel/intern/mask_rasterize.c
index 694f892f2c8..2c60f52578d 100644
--- a/source/blender/blenkernel/intern/mask_rasterize.c
+++ b/source/blender/blenkernel/intern/mask_rasterize.c
@@ -918,6 +918,10 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 			unsigned int face_index;
 			int scanfill_flag = 0;
 
+			bool is_isect = false;
+			ListBase isect_remvertbase = {NULL, NULL};
+			ListBase isect_remedgebase = {NULL, NULL};
+
 			/* now we have all the splines */
 			face_coords = MEM_mallocN((sizeof(float) * 3) * sf_vert_tot, "maskrast_face_coords");
 
@@ -941,12 +945,50 @@ void BKE_maskrasterize_handle_init(MaskRasterHandle *mr_handle, struct Mask *mas
 				cos += 3;
 			}
 
+
+			/* --- inefficient self-intersect case --- */
+			/* if self intersections are found, its too trickty to attempt to map vertices
+			 * so just realloc and add entirely new vertices - the result of the self-intersect check
+			 */
+			if ((masklay->flag & MASK_LAYERFLAG_FILL_OVERLAP) &&
+			    (is_isect = BLI_scanfill_calc_self_isect(&sf_ctx,
+			                                             &isect_remvertbase,
+			                                             &isect_remedgebase)))
+			{
+				unsigned int sf_vert_tot_isect = (unsigned int)BLI_countlist(&sf_ctx.fillvertbase);
+				unsigned int i = sf_vert_tot;
+
+				face_coords = MEM_reallocN(face_coords, sizeof(float[3]) * (sf_vert_tot + sf_vert_tot_isect));
+
+				cos = (float *)&face_coords[sf_vert_tot][0];
+
+				for (sf_vert = sf_ctx.fillvertbase.first; sf_vert; sf_vert = sf_vert->next) {
+					copy_v3_v3(cos, sf_vert->co);
+					sf_vert->tmp.u = i++;
+					cos += 3;
+				}
+
+				sf_vert_tot += sf_vert_tot_isect;
+
+				/* we need to calc polys after self intersect */
+				scanfill_flag |= BLI_SCANFILL_CALC_POLYS;
+			}
+			/* --- end inefficient code --- */
+
+
 			/* main scan-fill */
 			if ((masklay->flag & MASK_LAYERFLAG_FILL_DISCRETE) == 0)
 				scanfill_flag |= BLI_SCANFILL_CALC_HOLES;
 
 			sf_tri_tot = (unsigned int)BLI_scanfill_calc_ex(&sf_ctx, scanfill_flag, zvec);
 
+			if (is_isect) {
+				/* add removed data back, we only need edges for feather,
+				 * but add verts back so they get freed along with others */
+				BLI_movelisttolist(&sf_ctx.fillvertbase, &isect_remvertbase);
+				BLI_movelisttolist(&sf_ctx.filledgebase, &isect_remedgebase);
+			}
+
 			face_array = MEM_mallocN(sizeof(*face_array) * ((size_t)sf_tri_tot + (size_t)tot_feather_quads), "maskrast_face_index");
 			face_index = 0;