Fix for 2042

With the new rule that allows correct calculation of vertex normals on a
mixed solid/smooth mesh, it is essential that vertex normals get
recalculated when changing smooth settings. Such a facility doesnt exist
in Blender yet, only after leaving editmode.
Hacking in a "enter editmode, leave editmode" event on the "Set Smooth"
button isn't nice... instead I've tweaked the calculus of vertexnormals
that it always sets them OK, apart from where they get mixed with solid
faces. Only in rare occasions this can still go "wrong" and needs a
TAB-TAB to fix. Will add that comment in release notes. For next release
we should definitely solve this smoothing bizz!

2 files changed, 73 insertions, 61 deletions

diff --git a/source/blender/src/editmesh.c b/source/blender/src/editmesh.c
index e7b6812c73c..69a8320b638 100644
--- a/source/blender/src/editmesh.c
+++ b/source/blender/src/editmesh.c
@@ -923,6 +923,7 @@ void load_editMesh(void)
 	/* eed->f2==0: not in face, f2==1: draw it */
 	/* eed->f1 : flag for dynaface (cylindertest, old engine) */
 	/* eve->f1 : flag for dynaface (sphere test, old engine) */
+	/* eve->f2 : being used in vertexnormals */
 	edge_drawflags();
 	
 	/* this sets efa->puno, punoflag (for vertex normal & projection) */
diff --git a/source/blender/src/editmesh_lib.c b/source/blender/src/editmesh_lib.c
index f1fda764d9b..11370232e3d 100644
--- a/source/blender/src/editmesh_lib.c
+++ b/source/blender/src/editmesh_lib.c
@@ -1233,11 +1233,11 @@ void delfaceflag(int flag)
 
 /* ********************* */
 
-static int contrpuntnorm(float *n, float *puno)  /* dutch: check vertex normal */
+static int check_vnormal_flip(float *n, float *vnorm) 
 {
 	float inp;
 
-	inp= n[0]*puno[0]+n[1]*puno[1]+n[2]*puno[2];
+	inp= n[0]*vnorm[0]+n[1]*vnorm[1]+n[2]*vnorm[2];
 
 	/* angles 90 degrees: dont flip */
 	if(inp> -0.000001) return 0;
@@ -1254,7 +1254,7 @@ void vertexnormals(int testflip)
 	EditFace *efa;	
 	float n1[3], n2[3], n3[3], n4[3], co[4], fac1, fac2, fac3, fac4, *temp;
 	float *f1, *f2, *f3, *f4, xn, yn, zn;
-	float len;
+	float len, area;
 	
 	if(G.obedit && G.obedit->type==OB_MESH) {
 		me= G.obedit->data;
@@ -1274,84 +1274,95 @@ void vertexnormals(int testflip)
 		return;
 	}
 
-	/* clear normals */	
+	/* clear normals, clear flag */	
 	eve= em->verts.first;
 	while(eve) {
 		eve->no[0]= eve->no[1]= eve->no[2]= 0.0;
+		eve->f2= 0;
 		eve= eve->next;
 	}
 	
-	/* calculate cosine angles and add to vertex normal */
-	efa= em->faces.first;
-	while(efa) {
+	/* check for vertices being shared by both solid and smooth face, 
+	   these get vertexnormal of smooth face normal only */
+	for(efa= em->faces.first; efa; efa= efa->next) {
 		if(efa->flag & ME_SMOOTH) {
-			VecSubf(n1, efa->v2->co, efa->v1->co);
-			VecSubf(n2, efa->v3->co, efa->v2->co);
-			Normalise(n1);
-			Normalise(n2);
-
-			if(efa->v4==0) {
-				VecSubf(n3, efa->v1->co, efa->v3->co);
-				Normalise(n3);
-				
-				co[0]= saacos(-n3[0]*n1[0]-n3[1]*n1[1]-n3[2]*n1[2]);
-				co[1]= saacos(-n1[0]*n2[0]-n1[1]*n2[1]-n1[2]*n2[2]);
-				co[2]= saacos(-n2[0]*n3[0]-n2[1]*n3[1]-n2[2]*n3[2]);
-				
-			}
-			else {
-				VecSubf(n3, efa->v4->co, efa->v3->co);
-				VecSubf(n4, efa->v1->co, efa->v4->co);
-				Normalise(n3);
-				Normalise(n4);
-				
-				co[0]= saacos(-n4[0]*n1[0]-n4[1]*n1[1]-n4[2]*n1[2]);
-				co[1]= saacos(-n1[0]*n2[0]-n1[1]*n2[1]-n1[2]*n2[2]);
-				co[2]= saacos(-n2[0]*n3[0]-n2[1]*n3[1]-n2[2]*n3[2]);
-				co[3]= saacos(-n3[0]*n4[0]-n3[1]*n4[1]-n3[2]*n4[2]);
-			}
+			efa->v1->f2 |= 1; efa->v2->f2 |= 1; efa->v3->f2 |= 1;
+			if(efa->v4) efa->v4->f2 |= 1;
+		}
+		else {
+			efa->v1->f2 |= 2; efa->v2->f2 |= 2; efa->v3->f2 |= 2;
+			if(efa->v4) efa->v4->f2 |= 2;
+		}
+	}
+	
+	/* calculate cosine angles and add to vertex normal */
+	for(efa= em->faces.first; efa; efa= efa->next) {
+		VecSubf(n1, efa->v2->co, efa->v1->co);
+		VecSubf(n2, efa->v3->co, efa->v2->co);
+		Normalise(n1);
+		Normalise(n2);
+
+		if(efa->v4==0) {
+			VecSubf(n3, efa->v1->co, efa->v3->co);
+			Normalise(n3);
+			
+			//area= AreaT3Dfl(efa->v1->co, efa->v2->co, efa->v3->co);
+			//if(area!=0.0) area=1.0/area; 
+			//area= sqrt(area);
+			area= 1.0;
+			
+			co[0]= area*saacos(-n3[0]*n1[0]-n3[1]*n1[1]-n3[2]*n1[2]);
+			co[1]= area*saacos(-n1[0]*n2[0]-n1[1]*n2[1]-n1[2]*n2[2]);
+			co[2]= area*saacos(-n2[0]*n3[0]-n2[1]*n3[1]-n2[2]*n3[2]);
+			
+		}
+		else {
+			VecSubf(n3, efa->v4->co, efa->v3->co);
+			VecSubf(n4, efa->v1->co, efa->v4->co);
+			Normalise(n3);
+			Normalise(n4);
+			
+			//area= AreaQ3Dfl(efa->v1->co, efa->v2->co, efa->v3->co, efa->v4->co);
+			//if(area!=0.0) area=1.0/area; 
+			//area= sqrt(area);
+			area= 1.0;
 			
+			co[0]= area*saacos(-n4[0]*n1[0]-n4[1]*n1[1]-n4[2]*n1[2]);
+			co[1]= area*saacos(-n1[0]*n2[0]-n1[1]*n2[1]-n1[2]*n2[2]);
+			co[2]= area*saacos(-n2[0]*n3[0]-n2[1]*n3[1]-n2[2]*n3[2]);
+			co[3]= area*saacos(-n3[0]*n4[0]-n3[1]*n4[1]-n3[2]*n4[2]);
+		}
+		
+		if(efa->v1->f2!=3 || (efa->flag & ME_SMOOTH)) {
 			temp= efa->v1->no;
-			if(testflip && contrpuntnorm(efa->n, temp) ) co[0]= -co[0];
+			if(testflip && check_vnormal_flip(efa->n, temp) ) co[0]= -co[0];
 			temp[0]+= co[0]*efa->n[0];
 			temp[1]+= co[0]*efa->n[1];
 			temp[2]+= co[0]*efa->n[2];
-			
+		}
+		
+		if(efa->v2->f2!=3 || (efa->flag & ME_SMOOTH)) {
 			temp= efa->v2->no;
-			if(testflip && contrpuntnorm(efa->n, temp) ) co[1]= -co[1];
+			if(testflip && check_vnormal_flip(efa->n, temp) ) co[1]= -co[1];
 			temp[0]+= co[1]*efa->n[0];
 			temp[1]+= co[1]*efa->n[1];
 			temp[2]+= co[1]*efa->n[2];
-			
+		}
+		
+		if(efa->v3->f2!=3 || (efa->flag & ME_SMOOTH)) {
 			temp= efa->v3->no;
-			if(testflip && contrpuntnorm(efa->n, temp) ) co[2]= -co[2];
+			if(testflip && check_vnormal_flip(efa->n, temp) ) co[2]= -co[2];
 			temp[0]+= co[2]*efa->n[0];
 			temp[1]+= co[2]*efa->n[1];
 			temp[2]+= co[2]*efa->n[2];
-			
-			if(efa->v4) {
-				temp= efa->v4->no;
-				if(testflip && contrpuntnorm(efa->n, temp) ) co[3]= -co[3];
-				temp[0]+= co[3]*efa->n[0];
-				temp[1]+= co[3]*efa->n[1];
-				temp[2]+= co[3]*efa->n[2];
-			}
-		}		
-		efa= efa->next;
-	}
-	/* check solid faces */
-	for(efa= em->faces.first; efa; efa= efa->next) {
-		if((efa->flag & ME_SMOOTH)==0) {
-			f1= efa->v1->no;
-			if(f1[0]==0.0 && f1[1]==0.0 && f1[2]==0.0) VECCOPY(f1, efa->n);
-			f1= efa->v2->no;
-			if(f1[0]==0.0 && f1[1]==0.0 && f1[2]==0.0) VECCOPY(f1, efa->n);
-			f1= efa->v3->no;
-			if(f1[0]==0.0 && f1[1]==0.0 && f1[2]==0.0) VECCOPY(f1, efa->n);
-			if(efa->v4) {
-				f1= efa->v4->no;
-				if(f1[0]==0.0 && f1[1]==0.0 && f1[2]==0.0) VECCOPY(f1, efa->n);
-			}
+		}
+		
+		if(efa->v4 && (efa->v4->f2!=3 || (efa->flag & ME_SMOOTH))) {
+			temp= efa->v4->no;
+			if(testflip && check_vnormal_flip(efa->n, temp) ) co[3]= -co[3];
+			temp[0]+= co[3]*efa->n[0];
+			temp[1]+= co[3]*efa->n[1];
+			temp[2]+= co[3]*efa->n[2];
 		}
 	}