Fix #26582, #26586, #26613: recent normal calculation changes didn't take

into account that some tools use normals for things other than display. Now
we properly initialize vertex normals at flat faces too.

Also fixed a normal refresh issue, and deduplicated CDDM/mesh normal
calculation code.

8 files changed, 79 insertions, 87 deletions

diff --git a/source/blender/blenkernel/BKE_blender.h b/source/blender/blenkernel/BKE_blender.h
index 2bba079d605..6f8e9bbe8df 100644
--- a/source/blender/blenkernel/BKE_blender.h
+++ b/source/blender/blenkernel/BKE_blender.h
@@ -44,7 +44,7 @@ extern "C" {
  * and keep comment above the defines.
  * Use STRINGIFY() rather then defining with quotes */
 #define BLENDER_VERSION			256
-#define BLENDER_SUBVERSION		4
+#define BLENDER_SUBVERSION		5
 
 #define BLENDER_MINVERSION		250
 #define BLENDER_MINSUBVERSION	0
diff --git a/source/blender/blenkernel/BKE_mesh.h b/source/blender/blenkernel/BKE_mesh.h
index 4a7e14f7d7d..ae7ac0dabee 100644
--- a/source/blender/blenkernel/BKE_mesh.h
+++ b/source/blender/blenkernel/BKE_mesh.h
@@ -100,7 +100,7 @@ void mesh_strip_loose_edges(struct Mesh *me);
 	/* Calculate vertex and face normals, face normals are returned in *faceNors_r if non-NULL
 	 * and vertex normals are stored in actual mverts.
 	 */
-void mesh_calc_normals(struct MVert *mverts, int numVerts, struct MFace *mfaces, int numFaces, float **faceNors_r);
+void mesh_calc_normals(struct MVert *mverts, int numVerts, struct MFace *mfaces, int numFaces, float (*faceNors_r)[3]);
 
 	/* Return a newly MEM_malloc'd array of all the mesh vertex locations
 	 * (_numVerts_r_ may be NULL) */
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index b7034a7db8a..fc0d5d90687 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -1789,26 +1789,15 @@ void CDDM_apply_vert_normals(DerivedMesh *dm, short (*vertNormals)[3])
 		VECCOPY(vert->no, vertNormals[i]);
 }
 
-/* adapted from mesh_calc_normals */
 void CDDM_calc_normals(DerivedMesh *dm)
 {
 	CDDerivedMesh *cddm = (CDDerivedMesh*)dm;
-	float (*temp_nors)[3];
 	float (*face_nors)[3];
-	int i;
-	int numVerts = dm->numVertData;
-	int numFaces = dm->numFaceData;
-	MFace *mfaces;
-	MVert *mv;
-
-	if(numVerts == 0) return;
 
-	temp_nors = MEM_callocN(numVerts * sizeof(*temp_nors),
-							"CDDM_calc_normals temp_nors");
+	if(dm->numVertData == 0) return;
 
 	/* we don't want to overwrite any referenced layers */
-	mv = CustomData_duplicate_referenced_layer(&dm->vertData, CD_MVERT);
-	cddm->mvert = mv;
+	cddm->mvert = CustomData_duplicate_referenced_layer(&dm->vertData, CD_MVERT);
 
 	/* make a face normal layer if not present */
 	face_nors = CustomData_get_layer(&dm->faceData, CD_NORMAL);
@@ -1816,50 +1805,8 @@ void CDDM_calc_normals(DerivedMesh *dm)
 		face_nors = CustomData_add_layer(&dm->faceData, CD_NORMAL, CD_CALLOC,
 										 NULL, dm->numFaceData);
 
-	/* calculate face normals and add to vertex normals */
-	mfaces = CDDM_get_faces(dm);
-	for(i = 0; i < numFaces; i++) {
-		MFace * mf = &mfaces[i];
-		float *f_no = face_nors[i];
-
-		if(mf->v4)
-			normal_quad_v3(f_no, mv[mf->v1].co, mv[mf->v2].co, mv[mf->v3].co, mv[mf->v4].co);
-		else
-			normal_tri_v3(f_no, mv[mf->v1].co, mv[mf->v2].co, mv[mf->v3].co);
-		
-		if((mf->flag&ME_SMOOTH)!=0) {
-			float *n4 = (mf->v4)? temp_nors[mf->v4]: NULL;
-			float *c4 = (mf->v4)? mv[mf->v4].co: NULL;
-
-			accumulate_vertex_normals(temp_nors[mf->v1], temp_nors[mf->v2], temp_nors[mf->v3], n4,
-				f_no, mv[mf->v1].co, mv[mf->v2].co, mv[mf->v3].co, c4);
-		}
-	}
-
-	for(i = 0; i < numFaces; i++) {
-		MFace * mf = &mfaces[i];
-
-		if((mf->flag&ME_SMOOTH)==0) {
-			float *f_no = face_nors[i];
-
-			if(is_zero_v3(temp_nors[mf->v1])) copy_v3_v3(temp_nors[mf->v1], f_no);
-			if(is_zero_v3(temp_nors[mf->v2])) copy_v3_v3(temp_nors[mf->v2], f_no);
-			if(is_zero_v3(temp_nors[mf->v3])) copy_v3_v3(temp_nors[mf->v3], f_no);
-			if(mf->v4 && is_zero_v3(temp_nors[mf->v4])) copy_v3_v3(temp_nors[mf->v4], f_no);
-		}
-	}
-
-	/* normalize vertex normals and assign */
-	for(i = 0; i < numVerts; i++, mv++) {
-		float *no = temp_nors[i];
-		
-		if(normalize_v3(no) == 0.0f)
-			normalize_v3_v3(no, mv->co);
-
-		normal_float_to_short_v3(mv->no, no);
-	}
-	
-	MEM_freeN(temp_nors);
+	/* calculate face normals */
+	mesh_calc_normals(cddm->mvert, dm->numVertData, CDDM_get_faces(dm), dm->numFaceData, face_nors);
 }
 
 void CDDM_calc_edges(DerivedMesh *dm)
diff --git a/source/blender/blenkernel/intern/mesh.c b/source/blender/blenkernel/intern/mesh.c
index 1e29f5c9d3f..365817265f4 100644
--- a/source/blender/blenkernel/intern/mesh.c
+++ b/source/blender/blenkernel/intern/mesh.c
@@ -1270,42 +1270,69 @@ void mesh_set_smooth_flag(Object *meshOb, int enableSmooth)
 			mf->flag &= ~ME_SMOOTH;
 		}
 	}
+
+	mesh_calc_normals(me->mvert, me->totvert, me->mface, me->totface, NULL);
 }
 
-void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces, float **faceNors_r) 
+void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces, float (*faceNors_r)[3]) 
 {
 	float (*tnorms)[3]= MEM_callocN(numVerts*sizeof(*tnorms), "tnorms");
-	float *fnors= MEM_callocN(sizeof(*fnors)*3*numFaces, "meshnormals");
+	float (*fnors)[3]= (faceNors_r)? faceNors_r: MEM_callocN(sizeof(*fnors)*numFaces, "meshnormals");
 	int i;
+	int found_flat=0;
 
 	for(i=0; i<numFaces; i++) {
 		MFace *mf= &mfaces[i];
-		float *f_no= &fnors[i*3];
+		float *f_no= fnors[i];
 
 		if(mf->v4)
 			normal_quad_v3(f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co, mverts[mf->v4].co);
 		else
 			normal_tri_v3(f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co);
 
-		if((mf->flag&ME_SMOOTH)!=0) {
+		if(mf->flag & ME_SMOOTH) {
 			float *n4 = (mf->v4)? tnorms[mf->v4]: NULL;
 			float *c4 = (mf->v4)? mverts[mf->v4].co: NULL;
 
 			accumulate_vertex_normals(tnorms[mf->v1], tnorms[mf->v2], tnorms[mf->v3], n4,
 				f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co, c4);
 		}
+		else {
+			found_flat=1;
+		}
 	}
 
-	for(i=0; i<numFaces; i++) {
-		MFace *mf= &mfaces[i];
+	/* build smooth normals for uninitialized normals at faces set to flat */
+	if(found_flat!=0) {
+		const int nr_bits= sizeof(int)*8;
+		const int nr_words= (numVerts+(nr_bits-1))/nr_bits;
+		int *bit_array= (int*)MEM_callocN(sizeof(int)*numVerts*nr_words, "temp buffer");
+
+		for(i=0; i<numFaces; i++) {
+			MFace *mf= &mfaces[i];
 
-		if((mf->flag&ME_SMOOTH)==0) {
-			float *f_no= &fnors[i*3];
-			if(is_zero_v3(tnorms[mf->v1])) copy_v3_v3(tnorms[mf->v1], f_no);
-			if(is_zero_v3(tnorms[mf->v2])) copy_v3_v3(tnorms[mf->v2], f_no);
-			if(is_zero_v3(tnorms[mf->v3])) copy_v3_v3(tnorms[mf->v3], f_no);
-			if(mf->v4 && is_zero_v3(tnorms[mf->v4])) copy_v3_v3(tnorms[mf->v4], f_no);
+			if(!(mf->flag & ME_SMOOTH)) {
+				if(is_zero_v3(tnorms[mf->v1])) bit_array[mf->v1>>nr_bits]|=(1<<(mf->v1&(nr_bits-1)));
+				if(is_zero_v3(tnorms[mf->v2])) bit_array[mf->v2>>nr_bits]|=(1<<(mf->v2&(nr_bits-1)));
+				if(is_zero_v3(tnorms[mf->v3])) bit_array[mf->v3>>nr_bits]|=(1<<(mf->v3&(nr_bits-1)));
+				if(mf->v4 && is_zero_v3(tnorms[mf->v4])) bit_array[mf->v4>>nr_bits]|=(1<<(mf->v4&(nr_bits-1)));
+			}
 		}
+
+		for(i=0; i<numFaces; i++) {
+			MFace *mf= &mfaces[i];
+
+			if((mf->flag&ME_SMOOTH)==0) {
+				float *f_no= fnors[i];
+
+				if(bit_array[mf->v1>>nr_bits]&(1<<(mf->v1&(nr_bits-1)))) add_v3_v3(tnorms[mf->v1], f_no);
+				if(bit_array[mf->v2>>nr_bits]&(1<<(mf->v2&(nr_bits-1)))) add_v3_v3(tnorms[mf->v2], f_no);
+				if(bit_array[mf->v3>>nr_bits]&(1<<(mf->v3&(nr_bits-1)))) add_v3_v3(tnorms[mf->v3], f_no);
+				if(mf->v4 && bit_array[mf->v4>>nr_bits]&(1<<(mf->v4&(nr_bits-1)))) add_v3_v3(tnorms[mf->v4], f_no);
+			}
+		}
+
+		MEM_freeN(bit_array);
 	}
 
 	/* following Mesh convention; we use vertex coordinate itself for normal in this case */
@@ -1321,9 +1348,7 @@ void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces,
 	
 	MEM_freeN(tnorms);
 
-	if(faceNors_r)
-		*faceNors_r = fnors;
-	else
+	if(fnors != faceNors_r)
 		MEM_freeN(fnors);
 }
 
diff --git a/source/blender/blenlib/BLI_editVert.h b/source/blender/blenlib/BLI_editVert.h
index b422b79b165..ba745af5a0b 100644
--- a/source/blender/blenlib/BLI_editVert.h
+++ b/source/blender/blenlib/BLI_editVert.h
@@ -60,6 +60,7 @@ typedef struct EditVert
 		void            *p;
 		intptr_t         l;
 		float            fp;
+		int				 t;
 	} tmp;
 	float no[3]; /*vertex normal */
 	float co[3]; /*vertex location */
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 22ce3bd590a..647352ec727 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -11552,7 +11552,7 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
 		}
 	}
 
-	if (main->versionfile < 256 || (main->versionfile == 256 && main->subversionfile <4)){
+	if (main->versionfile < 256 || (main->versionfile == 256 && main->subversionfile < 5)){
 		Mesh *me;
 
 		for(me= main->mesh.first; me; me= me->id.next)
diff --git a/source/blender/editors/mesh/editmesh_lib.c b/source/blender/editors/mesh/editmesh_lib.c
index 384b311437b..db990e033df 100644
--- a/source/blender/editors/mesh/editmesh_lib.c
+++ b/source/blender/editors/mesh/editmesh_lib.c
@@ -2001,36 +2001,53 @@ void recalc_editnormals(EditMesh *em)
 {
 	EditFace *efa;
 	EditVert *eve;
+	int found_flat= 0;
 
-	for(eve= em->verts.first; eve; eve=eve->next) {
-		eve->no[0] = eve->no[1] = eve->no[2] = 0.0;
-	}
+	for(eve= em->verts.first; eve; eve=eve->next)
+		zero_v3(eve->no);
 
 	for(efa= em->faces.first; efa; efa=efa->next) {
 		if(efa->v4) {
-			normal_quad_v3( efa->n,efa->v1->co, efa->v2->co, efa->v3->co, efa->v4->co);
+			normal_quad_v3(efa->n, efa->v1->co, efa->v2->co, efa->v3->co, efa->v4->co);
 			cent_quad_v3(efa->cent, efa->v1->co, efa->v2->co, efa->v3->co, efa->v4->co);
 		}
 		else {
-			normal_tri_v3( efa->n,efa->v1->co, efa->v2->co, efa->v3->co);
+			normal_tri_v3(efa->n, efa->v1->co, efa->v2->co, efa->v3->co);
 			cent_tri_v3(efa->cent, efa->v1->co, efa->v2->co, efa->v3->co);
 		}
 
-		if((efa->flag&ME_SMOOTH)!=0) {
+		if(efa->flag & ME_SMOOTH) {
 			float *n4= (efa->v4)? efa->v4->no: NULL;
 			float *c4= (efa->v4)? efa->v4->co: NULL;
 
 			accumulate_vertex_normals(efa->v1->no, efa->v2->no, efa->v3->no, n4,
 				efa->n, efa->v1->co, efa->v2->co, efa->v3->co, c4);
 		}
+		else
+			found_flat= 1;
 	}
 
-	for(efa= em->faces.first; efa; efa=efa->next) {
-		if((efa->flag&ME_SMOOTH)==0) {
-			if(is_zero_v3(efa->v1->no)) copy_v3_v3(efa->v1->no, efa->n);
-			if(is_zero_v3(efa->v2->no)) copy_v3_v3(efa->v2->no, efa->n);
-			if(is_zero_v3(efa->v3->no)) copy_v3_v3(efa->v3->no, efa->n);
-			if(efa->v4 && is_zero_v3(efa->v4->no)) copy_v3_v3(efa->v4->no, efa->n);
+	/* build smooth normals for uninitialized normals at faces set to flat */
+	if(found_flat!=0) {
+		for(efa= em->faces.first; efa; efa=efa->next) {
+			efa->v1->tmp.t= 0;
+			efa->v2->tmp.t= 0;
+			efa->v3->tmp.t= 0;
+			if(efa->v4) efa->v4->tmp.t= 0;
+
+			if(!(efa->flag & ME_SMOOTH)) {
+				if(is_zero_v3(efa->v1->no)) efa->v1->tmp.t= 1;
+				if(is_zero_v3(efa->v2->no)) efa->v2->tmp.t= 1;
+				if(is_zero_v3(efa->v3->no)) efa->v3->tmp.t= 1;
+				if(efa->v4 && is_zero_v3(efa->v4->no)) efa->v4->tmp.t= 1;
+			}
+		}
+
+		for(efa= em->faces.first; efa; efa=efa->next) {
+			if(efa->v1->tmp.t) add_v3_v3(efa->v1->no, efa->n);
+			if(efa->v2->tmp.t) add_v3_v3(efa->v2->no, efa->n);
+			if(efa->v3->tmp.t) add_v3_v3(efa->v3->no, efa->n);
+			if(efa->v4 && efa->v4->tmp.t) add_v3_v3(efa->v4->no, efa->n);
 		}
 	}
 
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 5525a486438..40387b82205 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -7451,6 +7451,8 @@ static void mesh_set_smooth_faces(EditMesh *em, short smooth)
 			else efa->flag &= ~ME_SMOOTH;
 		}
 	}
+
+	recalc_editnormals(em);
 }
 
 static int mesh_faces_shade_smooth_exec(bContext *C, wmOperator *UNUSED(op))