Bug fix #2629

Aye... OpenGL cannot draw concave (C shaped) polygons... that screws up the
Lasso tool, when it uses backbuffer selection.
Examined for little while the GLU Tesselation library, but apart from its
nightmarish structure, it's even stupid (no builtin clock/counterclock).

So, instead coded a DispList based function using Blender's edgefill.
Works like a charm! :)

3 files changed, 52 insertions, 15 deletions

diff --git a/source/blender/blenkernel/BKE_displist.h b/source/blender/blenkernel/BKE_displist.h
index ebae7d9bd31..3193c8cbfec 100644
--- a/source/blender/blenkernel/BKE_displist.h
+++ b/source/blender/blenkernel/BKE_displist.h
@@ -147,6 +147,7 @@ void boundbox_displist(struct Object *ob);
 void imagestodisplist(void);
 void reshadeall_displist(void);
 void test_all_displists(void);
+void filldisplist(struct ListBase *dispbase, struct ListBase *to);
 
 #endif
 
diff --git a/source/blender/blenkernel/intern/displist.c b/source/blender/blenkernel/intern/displist.c
index 374f22770a2..9ad4fa89278 100644
--- a/source/blender/blenkernel/intern/displist.c
+++ b/source/blender/blenkernel/intern/displist.c
@@ -1353,7 +1353,7 @@ static void curve_to_displist(ListBase *nubase, ListBase *dispbase)
 }
 
 
-static void filldisplist(ListBase *dispbase, ListBase *to)
+void filldisplist(ListBase *dispbase, ListBase *to)
 {
 	EditVert *eve, *v1, *vlast;
 	EditFace *efa;
@@ -1365,11 +1365,6 @@ static void filldisplist(ListBase *dispbase, ListBase *to)
 	if(dispbase==0) return;
 	if(dispbase->first==0) return;
 
-	/* tijd= clock(); */
-	/* bit-wise and comes after == .... so this doesn't work...  */
-/*  	if(G.f & G_PLAYANIM == 0) waitcursor(1); */
-	if( !(G.f & G_PLAYANIM) ) waitcursor(1);
-
 	while(cont) {
 		cont= 0;
 		totvert=0;
@@ -1473,12 +1468,6 @@ static void filldisplist(ListBase *dispbase, ListBase *to)
 	}
 	
 	/* do not free polys, needed for wireframe display */
-	
-	/* same as above ... */
-/*  	if(G.f & G_PLAYANIM == 0) waitcursor(0); */
-	if( !(G.f & G_PLAYANIM) ) waitcursor(0);
-	/* printf("time: %d\n",(clock()-tijd)/1000); */
-
 }
 
 static void bevels_to_filledpoly(Curve *cu, ListBase *dispbase)
diff --git a/source/blender/src/editmesh_mods.c b/source/blender/src/editmesh_mods.c
index 3e959aef2bc..3e19cd0bc18 100644
--- a/source/blender/src/editmesh_mods.c
+++ b/source/blender/src/editmesh_mods.c
@@ -237,6 +237,52 @@ static unsigned int sample_backbuf_rect(unsigned int *buf, int size, int min, in
 /* facilities for border select and circle select */
 static char *selbuf= NULL;
 
+/* opengl doesn't support concave... */
+static void draw_triangulated(short mcords[][2], short tot)
+{
+	ListBase lb={NULL, NULL};
+	DispList *dl;
+	float *fp;
+	int a;
+	
+	/* make displist */
+	dl= MEM_callocN(sizeof(DispList), "poly disp");
+	dl->type= DL_POLY;
+	dl->parts= 1;
+	dl->nr= tot;
+	dl->verts= fp=  MEM_callocN(tot*3*sizeof(float), "poly verts");
+	BLI_addtail(&lb, dl);
+	
+	for(a=0; a<tot; a++, fp+=3) {
+		fp[0]= (float)mcords[a][0];
+		fp[1]= (float)mcords[a][1];
+	}
+	
+	/* do the fill */
+	filldisplist(&lb, &lb);
+
+	/* do the draw */
+	dl= lb.first;	// filldisplist adds in head of list
+	if(dl->type==DL_INDEX3) {
+		int *index;
+		
+		a= dl->parts;
+		fp= dl->verts;
+		index= dl->index;
+		glBegin(GL_TRIANGLES);
+		while(a--) {
+			glVertex3fv(fp+3*index[0]);
+			glVertex3fv(fp+3*index[1]);
+			glVertex3fv(fp+3*index[2]);
+			index+= 3;
+		}
+		glEnd();
+	}
+	
+	freedisplist(&lb);
+}
+
+
 /* reads rect, and builds selection array for quick lookup */
 /* returns if all is OK */
 int EM_init_backbuf_border(short xmin, short ymin, short xmax, short ymax)
@@ -307,9 +353,10 @@ int EM_mask_init_backbuf_border(short mcords[][2], short tot, short xmin, short
 	
 	persp(PERSP_WIN);
 	glColor3ub(0, 0, 0);
-	glBegin(GL_POLYGON);
-	for(a=0; a<tot; a++) glVertex2s(mcords[a][0], mcords[a][1]);
-	glEnd();
+	
+	/* yah, opengl doesn't do concave... tsk! */
+ 	draw_triangulated(mcords, tot);	
+	
 	glBegin(GL_LINE_LOOP);	// for zero sized masks, lines
 	for(a=0; a<tot; a++) glVertex2s(mcords[a][0], mcords[a][1]);
 	glEnd();