Fix #26582, #26586, #26613: recent normal calculation changes didn't take

into account that some tools use normals for things other than display. Now
we properly initialize vertex normals at flat faces too.

Also fixed a normal refresh issue, and deduplicated CDDM/mesh normal
calculation code.

1 files changed, 40 insertions, 15 deletions

diff --git a/source/blender/blenkernel/intern/mesh.c b/source/blender/blenkernel/intern/mesh.c
index 1e29f5c9d3f..365817265f4 100644
--- a/source/blender/blenkernel/intern/mesh.c
+++ b/source/blender/blenkernel/intern/mesh.c
@@ -1270,42 +1270,69 @@ void mesh_set_smooth_flag(Object *meshOb, int enableSmooth)
 			mf->flag &= ~ME_SMOOTH;
 		}
 	}
+
+	mesh_calc_normals(me->mvert, me->totvert, me->mface, me->totface, NULL);
 }
 
-void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces, float **faceNors_r) 
+void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces, float (*faceNors_r)[3]) 
 {
 	float (*tnorms)[3]= MEM_callocN(numVerts*sizeof(*tnorms), "tnorms");
-	float *fnors= MEM_callocN(sizeof(*fnors)*3*numFaces, "meshnormals");
+	float (*fnors)[3]= (faceNors_r)? faceNors_r: MEM_callocN(sizeof(*fnors)*numFaces, "meshnormals");
 	int i;
+	int found_flat=0;
 
 	for(i=0; i<numFaces; i++) {
 		MFace *mf= &mfaces[i];
-		float *f_no= &fnors[i*3];
+		float *f_no= fnors[i];
 
 		if(mf->v4)
 			normal_quad_v3(f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co, mverts[mf->v4].co);
 		else
 			normal_tri_v3(f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co);
 
-		if((mf->flag&ME_SMOOTH)!=0) {
+		if(mf->flag & ME_SMOOTH) {
 			float *n4 = (mf->v4)? tnorms[mf->v4]: NULL;
 			float *c4 = (mf->v4)? mverts[mf->v4].co: NULL;
 
 			accumulate_vertex_normals(tnorms[mf->v1], tnorms[mf->v2], tnorms[mf->v3], n4,
 				f_no, mverts[mf->v1].co, mverts[mf->v2].co, mverts[mf->v3].co, c4);
 		}
+		else {
+			found_flat=1;
+		}
 	}
 
-	for(i=0; i<numFaces; i++) {
-		MFace *mf= &mfaces[i];
+	/* build smooth normals for uninitialized normals at faces set to flat */
+	if(found_flat!=0) {
+		const int nr_bits= sizeof(int)*8;
+		const int nr_words= (numVerts+(nr_bits-1))/nr_bits;
+		int *bit_array= (int*)MEM_callocN(sizeof(int)*numVerts*nr_words, "temp buffer");
+
+		for(i=0; i<numFaces; i++) {
+			MFace *mf= &mfaces[i];
 
-		if((mf->flag&ME_SMOOTH)==0) {
-			float *f_no= &fnors[i*3];
-			if(is_zero_v3(tnorms[mf->v1])) copy_v3_v3(tnorms[mf->v1], f_no);
-			if(is_zero_v3(tnorms[mf->v2])) copy_v3_v3(tnorms[mf->v2], f_no);
-			if(is_zero_v3(tnorms[mf->v3])) copy_v3_v3(tnorms[mf->v3], f_no);
-			if(mf->v4 && is_zero_v3(tnorms[mf->v4])) copy_v3_v3(tnorms[mf->v4], f_no);
+			if(!(mf->flag & ME_SMOOTH)) {
+				if(is_zero_v3(tnorms[mf->v1])) bit_array[mf->v1>>nr_bits]|=(1<<(mf->v1&(nr_bits-1)));
+				if(is_zero_v3(tnorms[mf->v2])) bit_array[mf->v2>>nr_bits]|=(1<<(mf->v2&(nr_bits-1)));
+				if(is_zero_v3(tnorms[mf->v3])) bit_array[mf->v3>>nr_bits]|=(1<<(mf->v3&(nr_bits-1)));
+				if(mf->v4 && is_zero_v3(tnorms[mf->v4])) bit_array[mf->v4>>nr_bits]|=(1<<(mf->v4&(nr_bits-1)));
+			}
 		}
+
+		for(i=0; i<numFaces; i++) {
+			MFace *mf= &mfaces[i];
+
+			if((mf->flag&ME_SMOOTH)==0) {
+				float *f_no= fnors[i];
+
+				if(bit_array[mf->v1>>nr_bits]&(1<<(mf->v1&(nr_bits-1)))) add_v3_v3(tnorms[mf->v1], f_no);
+				if(bit_array[mf->v2>>nr_bits]&(1<<(mf->v2&(nr_bits-1)))) add_v3_v3(tnorms[mf->v2], f_no);
+				if(bit_array[mf->v3>>nr_bits]&(1<<(mf->v3&(nr_bits-1)))) add_v3_v3(tnorms[mf->v3], f_no);
+				if(mf->v4 && bit_array[mf->v4>>nr_bits]&(1<<(mf->v4&(nr_bits-1)))) add_v3_v3(tnorms[mf->v4], f_no);
+			}
+		}
+
+		MEM_freeN(bit_array);
 	}
 
 	/* following Mesh convention; we use vertex coordinate itself for normal in this case */
@@ -1321,9 +1348,7 @@ void mesh_calc_normals(MVert *mverts, int numVerts, MFace *mfaces, int numFaces,
 	
 	MEM_freeN(tnorms);
 
-	if(faceNors_r)
-		*faceNors_r = fnors;
-	else
+	if(fnors != faceNors_r)
 		MEM_freeN(fnors);
 }