Two fixes;

- The function "convex()" in editmesh_lib() actually did not deliver
a proper test for convex at all. It was checking only if a quad could
be subdivided into 2 trias. Code for adding face (FKEY) used this
call in total confusing manner. That code was there in 1.40 already,
cannot find any clue what it was supposed todo... :)
Recoded convex() to deliver a proper test. FKEY will give warning on
attempt to make convex faces now.

- Added undo-free for editmode undo on file load

8 files changed, 102 insertions, 73 deletions

diff --git a/source/blender/blenlib/BLI_arithb.h b/source/blender/blenlib/BLI_arithb.h
index 4df5db5f39e..9935932ca31 100644
--- a/source/blender/blenlib/BLI_arithb.h
+++ b/source/blender/blenlib/BLI_arithb.h
@@ -540,6 +540,18 @@ VecRotToMat3(
 	float phi, 
 	float mat[][3]
 );
+
+/* intersect Line-Line
+	return:
+	-1: colliniar
+	 0: no intersection of segments
+	 1: exact intersection of segments
+	 2: cross-intersection of segments
+*/
+extern short IsectLL2Df(float *v1, float *v2, float *v3, float *v4);
+extern short IsectLL2Ds(short *v1, short *v2, short *v3, short *v4);
+
+
 	float *
 vectoquat(
 	float *vec, 
diff --git a/source/blender/blenlib/intern/arithb.c b/source/blender/blenlib/intern/arithb.c
index ff057a61e51..08a26821ccd 100644
--- a/source/blender/blenlib/intern/arithb.c
+++ b/source/blender/blenlib/intern/arithb.c
@@ -1874,6 +1874,56 @@ float AreaPoly3Dfl(int nr, float *verts, float *normal)
 	return (float)fabs(0.5*area/max);
 }
 
+/* intersect Line-Line, shorts */
+short IsectLL2Ds(short *v1, short *v2, short *v3, short *v4)
+{
+	/* return:
+	-1: colliniar
+	 0: no intersection of segments
+	 1: exact intersection of segments
+	 2: cross-intersection of segments
+	*/
+	float div, labda, mu;
+	
+	div= (v2[0]-v1[0])*(v4[1]-v3[1])-(v2[1]-v1[1])*(v4[0]-v3[0]);
+	if(div==0.0) return -1;
+	
+	labda= ((float)(v1[1]-v3[1])*(v4[0]-v3[0])-(v1[0]-v3[0])*(v4[1]-v3[1]))/div;
+	
+	mu= ((float)(v1[1]-v3[1])*(v2[0]-v1[0])-(v1[0]-v3[0])*(v2[1]-v1[1]))/div;
+	
+	if(labda>=0.0 && labda<=1.0 && mu>=0.0 && mu<=1.0) {
+		if(labda==0.0 || labda==1.0 || mu==0.0 || mu==1.0) return 1;
+		return 2;
+	}
+	return 0;
+}
+
+/* intersect Line-Line, floats */
+short IsectLL2Df(float *v1, float *v2, float *v3, float *v4)
+{
+	/* return:
+	-1: colliniar
+0: no intersection of segments
+1: exact intersection of segments
+2: cross-intersection of segments
+	*/
+	float div, labda, mu;
+	
+	div= (v2[0]-v1[0])*(v4[1]-v3[1])-(v2[1]-v1[1])*(v4[0]-v3[0]);
+	if(div==0.0) return -1;
+	
+	labda= ((float)(v1[1]-v3[1])*(v4[0]-v3[0])-(v1[0]-v3[0])*(v4[1]-v3[1]))/div;
+	
+	mu= ((float)(v1[1]-v3[1])*(v2[0]-v1[0])-(v1[0]-v3[0])*(v2[1]-v1[1]))/div;
+	
+	if(labda>=0.0 && labda<=1.0 && mu>=0.0 && mu<=1.0) {
+		if(labda==0.0 || labda==1.0 || mu==0.0 || mu==1.0) return 1;
+		return 2;
+	}
+	return 0;
+}
+
 void MinMax3(float *min, float *max, float *vec)
 {
 	if(min[0]>vec[0]) min[0]= vec[0];
diff --git a/source/blender/include/editmesh.h b/source/blender/include/editmesh.h
index 4d40133317b..02834c71e5d 100644
--- a/source/blender/include/editmesh.h
+++ b/source/blender/include/editmesh.h
@@ -84,7 +84,7 @@ extern void delfaceflag(int flag);
 extern void rotateflag(short flag, float *cent, float rotmat[][3]);
 extern void translateflag(short flag, float *vec);
 
-extern float convex(float *v1, float *v2, float *v3, float *v4);
+extern int convex(float *v1, float *v2, float *v3, float *v4);
 
 /* ******************* editmesh_mods.c */
 extern EditEdge *findnearestedge(short *dist);
diff --git a/source/blender/src/editmesh_add.c b/source/blender/src/editmesh_add.c
index 6fb6f3ddf3f..7b0388226cd 100644
--- a/source/blender/src/editmesh_add.c
+++ b/source/blender/src/editmesh_add.c
@@ -294,7 +294,6 @@ void addedgeface_mesh(void)
 	EditVert *eve, *neweve[4];
 	EditEdge *eed;
 	EditFace *efa;
-	float con1, con2, con3;
 	short amount=0;
 
 	if( (G.vd->lay & G.obedit->lay)==0 ) return;
@@ -352,18 +351,12 @@ void addedgeface_mesh(void)
 		
 			if(tria==2) join_triangles();
 			else {
-				con1= convex(neweve[0]->co, neweve[1]->co, neweve[2]->co, neweve[3]->co);
-				con2= convex(neweve[0]->co, neweve[2]->co, neweve[3]->co, neweve[1]->co);
-				con3= convex(neweve[0]->co, neweve[3]->co, neweve[1]->co, neweve[2]->co);
-
-				if(con1>=con2 && con1>=con3)
-					efa= addfacelist(neweve[0], neweve[1], neweve[2], neweve[3], NULL, NULL);
-				else if(con2>=con1 && con2>=con3)
-					efa= addfacelist(neweve[0], neweve[2], neweve[3], neweve[1], NULL, NULL);
-				else 
-					efa= addfacelist(neweve[0], neweve[2], neweve[1], neweve[3], NULL, NULL);
 				
-				EM_select_face(efa, 1);
+				if( convex(neweve[0]->co, neweve[1]->co, neweve[2]->co, neweve[3]->co) ) {
+					efa= addfacelist(neweve[0], neweve[1], neweve[2], neweve[3], NULL, NULL);
+					EM_select_face(efa, 1);
+				}
+				else error("The selected vertices form a concave quad");
 			}
 
 		}
diff --git a/source/blender/src/editmesh_lib.c b/source/blender/src/editmesh_lib.c
index 1e9f42615c1..6c17f14c11c 100644
--- a/source/blender/src/editmesh_lib.c
+++ b/source/blender/src/editmesh_lib.c
@@ -1483,33 +1483,38 @@ EditFace *exist_face(EditVert *v1, EditVert *v2, EditVert *v3, EditVert *v4)
 	return NULL;
 }
 
-
-float convex(float *v1, float *v2, float *v3, float *v4)
+/* evaluate if entire quad is a proper convex quad */
+int convex(float *v1, float *v2, float *v3, float *v4)
 {
-	float cross[3], test[3];
-	float inpr;
-	
-	/* make sure we do not test a (close to) zero sized tria in quad */
-	test[0]= AreaT3Dfl(v1, v2, v3);
-	test[1]= AreaT3Dfl(v2, v3, v4);
-	test[2]= AreaT3Dfl(v3, v4, v1);
-	
-	if(test[0]<test[1] && test[0]<test[2]) {
-		CalcNormFloat(v2, v3, v4, cross);
-		CalcNormFloat(v2, v4, v1, test);
-	}
-	else if(test[1]<test[0] && test[1]<test[2]) {
-		CalcNormFloat(v3, v4, v1, cross);
-		CalcNormFloat(v3, v1, v2, test);
+	float nor[3], vec[4][2];
+	
+	/* define projection */
+	CalcNormFloat4(v1, v2, v3, v4, nor);
+	nor[0]= ABS(nor[0]);
+	nor[1]= ABS(nor[1]);
+	nor[2]= ABS(nor[2]);
+	if(nor[2] >= nor[0] && nor[2]>= nor[1]) {
+		vec[0][0]= v1[0]; vec[0][1]= v1[1];
+		vec[1][0]= v2[0]; vec[1][1]= v2[1];
+		vec[2][0]= v3[0]; vec[2][1]= v3[1];
+		vec[3][0]= v4[0]; vec[3][1]= v4[1];
+	}
+	else if(nor[1] >= nor[0] && nor[1]>= nor[2]) {
+		vec[0][0]= v1[0]; vec[0][1]= v1[2];
+		vec[1][0]= v2[0]; vec[1][1]= v2[2];
+		vec[2][0]= v3[0]; vec[2][1]= v3[2];
+		vec[3][0]= v4[0]; vec[3][1]= v4[2];
 	}
-	else  {
-		CalcNormFloat(v4, v1, v2, cross);
-		CalcNormFloat(v4, v2, v3, test);
+	else {
+		vec[0][0]= v1[1]; vec[0][1]= v1[2];
+		vec[1][0]= v2[1]; vec[1][1]= v2[2];
+		vec[2][0]= v3[1]; vec[2][1]= v3[2];
+		vec[3][0]= v4[1]; vec[3][1]= v4[2];
 	}
 	
-	inpr= cross[0]*test[0]+cross[1]*test[1]+cross[2]*test[2];
-
-	return inpr;
+	/* linetests, the 2 diagonals have to instersect to be convex */
+	if( IsectLL2Df(vec[0], vec[2], vec[1], vec[3]) > 0 ) return 1;
+	return 0;
 }
 
 
diff --git a/source/blender/src/editmesh_tools.c b/source/blender/src/editmesh_tools.c
index 16a955e03c3..2a7a9fca72b 100644
--- a/source/blender/src/editmesh_tools.c
+++ b/source/blender/src/editmesh_tools.c
@@ -2052,7 +2052,7 @@ void beauty_fill(void)
                 if(ok) {
                     /* test convex */
                     givequadverts(efaa[0], efaa[1], &v1, &v2, &v3, &v4, uv, col);
-                    if( convex(v1->co, v2->co, v3->co, v4->co) > -0.5) {
+                    if( convex(v1->co, v2->co, v3->co, v4->co) ) {
 
                         /* test edges */
                         if( ((long)v1) > ((long)v3) ) {
@@ -2226,7 +2226,7 @@ void join_triangles(void)
 		1-----2        1-----2
 */
 				/* make new faces */
-				if( convex(v1->co, v2->co, v3->co, v4->co) > 0.01) {
+				if( convex(v1->co, v2->co, v3->co, v4->co) ) {
 					if(exist_face(v1, v2, v3, v4)==0) {
 						w = addfacelist(v1, v2, v3, v4, efaa[0], NULL); /* seam edge may get broken */
 						w->f= efaa[0]->f;	/* copy selection flag */
@@ -2322,7 +2322,7 @@ void edge_flip(void)
 */
 				/* make new faces */
 				if (v1 && v2 && v3){
-					if( convex(v1->co, v2->co, v3->co, v4->co) > 0.01) {
+					if( convex(v1->co, v2->co, v3->co, v4->co) ) {
 						if(exist_face(v1, v2, v3, v4)==0) {
 							w = addfacelist(v1, v2, v3, 0, efaa[1], NULL); /* outch this may break seams */ 
 							EM_select_face(w, 1);
@@ -2962,7 +2962,6 @@ void bevel_mesh(float bsize, int allfaces)
 	EditFace *efa, *example; //, *nextvl;
 	EditEdge *eed, *eed2;
 	EditVert *neweve[1024], *eve, *eve2, *eve3, *v1, *v2, *v3, *v4; //, *eve4;
-	float con1, con2, con3;
 	//short found4,  search;
 	//float f1, f2, f3, f4;
 	float cent[3], min[3], max[3];
@@ -3306,15 +3305,7 @@ void bevel_mesh(float bsize, int allfaces)
 					}
 				} else if (a==4) {
 					if(exist_face(neweve[0], neweve[1], neweve[2], neweve[3])==0) {
-						con1= convex(neweve[0]->co, neweve[1]->co, neweve[2]->co, neweve[3]->co);
-						con2= convex(neweve[0]->co, neweve[2]->co, neweve[3]->co, neweve[1]->co);
-						con3= convex(neweve[0]->co, neweve[3]->co, neweve[1]->co, neweve[2]->co);
-						if(con1>=con2 && con1>=con3)
-							efa= addfacelist(neweve[0], neweve[1], neweve[2], neweve[3], example,NULL);
-						else if(con2>=con1 && con2>=con3)
-							efa= addfacelist(neweve[0], neweve[2], neweve[3], neweve[1], example,NULL);
-						else 
-							efa= addfacelist(neweve[0], neweve[2], neweve[1], neweve[3], example,NULL);
+						efa= addfacelist(neweve[0], neweve[1], neweve[2], neweve[3], example,NULL);
 					}				
 				}
 				else if (a==3) {
diff --git a/source/blender/src/editview.c b/source/blender/src/editview.c
index 5522059ed35..9024589f497 100644
--- a/source/blender/src/editview.c
+++ b/source/blender/src/editview.c
@@ -209,30 +209,6 @@ static int lasso_inside(short mcords[][2], short moves, short sx, short sy)
 	return 0;
 }
 
-static short IsectLL2Ds(short *v1, short *v2, short *v3, short *v4)  /* intersect Line-Line */
-{
-	/* return:
-	-1: colliniar
-	 0: no intersection of segments
-	 1: exact intersection of segments
-	 2: cross-intersection of segments
-	*/
-	float div, labda, mu;
-	
-	div= (v2[0]-v1[0])*(v4[1]-v3[1])-(v2[1]-v1[1])*(v4[0]-v3[0]);
-	if(div==0.0) return -1;
-	
-	labda= ((float)(v1[1]-v3[1])*(v4[0]-v3[0])-(v1[0]-v3[0])*(v4[1]-v3[1]))/div;
-
-	mu= ((float)(v1[1]-v3[1])*(v2[0]-v1[0])-(v1[0]-v3[0])*(v2[1]-v1[1]))/div;
-	
-	if(labda>=0.0 && labda<=1.0 && mu>=0.0 && mu<=1.0) {
-		if(labda==0.0 || labda==1.0 || mu==0.0 || mu==1.0) return 1;
-		return 2;
-	}
-	return 0;
-}
-
 /* edge version for lasso select. we assume boundbox check was done */
 static int lasso_inside_edge(short mcords[][2], short moves, short *v1, short *v2)
 {
diff --git a/source/blender/src/usiblender.c b/source/blender/src/usiblender.c
index c3f5e472ef5..af6b8e1be02 100644
--- a/source/blender/src/usiblender.c
+++ b/source/blender/src/usiblender.c
@@ -146,6 +146,7 @@ void BIF_read_file(char *name)
 
 	winqueue_break= 1;	/* leave queues everywhere */
 
+	undo_editmode_clear();
 	BKE_reset_undo();
 	BKE_write_undo("original");	/* save current state */
 }
@@ -258,6 +259,7 @@ int BIF_read_homefile(void)
 		space_set_commmandline_options();
 
 		if (U.undosteps==0) U.undosteps=32;
+		undo_editmode_clear();
 		BKE_reset_undo();
 		BKE_write_undo("original");	/* save current state */