Fix #32867: normal map baking issue with flat shaded faces since bmesh. Also

removed the old unused normal map tangent computation code.

4 files changed, 21 insertions, 202 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 84691362a93..eb42ecd8b3a 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -2492,15 +2492,11 @@ static void SetTSpace(const SMikkTSpaceContext *pContext, const float fvTangent[
 void DM_add_tangent_layer(DerivedMesh *dm)
 {
 	/* mesh vars */
-	MTFace *mtface, *tf;
-	MFace *mface, *mf;
-	MVert *mvert, *v1, *v2, *v3, *v4;
-	MemArena *arena = NULL;
-	VertexTangent **vtangents = NULL;
+	MVert *mvert;
+	MTFace *mtface;
+	MFace *mface;
 	float (*orco)[3] = NULL, (*tangent)[4];
-	float *uv1, *uv2, *uv3, *uv4, *vtang;
-	float fno[3], tang[3], uv[4][2];
-	int i, j, len, mf_vi[4], totvert, totface, iCalcNewMethod;
+	int totvert, totface;
 	float *nors;
 
 	if (CustomData_get_layer_index(&dm->faceData, CD_TANGENT) != -1)
@@ -2526,14 +2522,8 @@ void DM_add_tangent_layer(DerivedMesh *dm)
 	DM_add_tessface_layer(dm, CD_TANGENT, CD_CALLOC, NULL);
 	tangent = DM_get_tessface_data_layer(dm, CD_TANGENT);
 	
-	/* allocate some space */
-	arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, "tangent layer arena");
-	BLI_memarena_use_calloc(arena);
-	vtangents = MEM_callocN(sizeof(VertexTangent *) * totvert, "VertexTangent");
-
 	/* new computation method */
-	iCalcNewMethod = 1;
-	if (iCalcNewMethod != 0) {
+	{
 		SGLSLMeshToTangent mesh2tangent = {0};
 		SMikkTSpaceContext sContext = {0};
 		SMikkTSpaceInterface sInterface = {0};
@@ -2556,82 +2546,8 @@ void DM_add_tangent_layer(DerivedMesh *dm)
 		sInterface.m_setTSpaceBasic = SetTSpace;
 
 		/* 0 if failed */
-		iCalcNewMethod = genTangSpaceDefault(&sContext);
+		genTangSpaceDefault(&sContext);
 	}
-
-	if (!iCalcNewMethod) {
-		/* sum tangents at connected vertices */
-		for (i = 0, tf = mtface, mf = mface; i < totface; mf++, tf++, i++) {
-			v1 = &mvert[mf->v1];
-			v2 = &mvert[mf->v2];
-			v3 = &mvert[mf->v3];
-
-			if (mf->v4) {
-				v4 = &mvert[mf->v4];
-				normal_quad_v3(fno, v4->co, v3->co, v2->co, v1->co);
-			}
-			else {
-				v4 = NULL;
-				normal_tri_v3(fno, v3->co, v2->co, v1->co);
-			}
-		
-			if (mtface) {
-				uv1 = tf->uv[0];
-				uv2 = tf->uv[1];
-				uv3 = tf->uv[2];
-				uv4 = tf->uv[3];
-			}
-			else {
-				uv1 = uv[0]; uv2 = uv[1]; uv3 = uv[2]; uv4 = uv[3];
-				map_to_sphere(&uv[0][0], &uv[0][1], orco[mf->v1][0], orco[mf->v1][1], orco[mf->v1][2]);
-				map_to_sphere(&uv[1][0], &uv[1][1], orco[mf->v2][0], orco[mf->v2][1], orco[mf->v2][2]);
-				map_to_sphere(&uv[2][0], &uv[2][1], orco[mf->v3][0], orco[mf->v3][1], orco[mf->v3][2]);
-				if (v4)
-					map_to_sphere(&uv[3][0], &uv[3][1], orco[mf->v4][0], orco[mf->v4][1], orco[mf->v4][2]);
-			}
-		
-			tangent_from_uv(uv1, uv2, uv3, v1->co, v2->co, v3->co, fno, tang);
-			sum_or_add_vertex_tangent(arena, &vtangents[mf->v1], tang, uv1);
-			sum_or_add_vertex_tangent(arena, &vtangents[mf->v2], tang, uv2);
-			sum_or_add_vertex_tangent(arena, &vtangents[mf->v3], tang, uv3);
-		
-			if (mf->v4) {
-				v4 = &mvert[mf->v4];
-			
-				tangent_from_uv(uv1, uv3, uv4, v1->co, v3->co, v4->co, fno, tang);
-				sum_or_add_vertex_tangent(arena, &vtangents[mf->v1], tang, uv1);
-				sum_or_add_vertex_tangent(arena, &vtangents[mf->v3], tang, uv3);
-				sum_or_add_vertex_tangent(arena, &vtangents[mf->v4], tang, uv4);
-			}
-		}
-	
-		/* write tangent to layer */
-		for (i = 0, tf = mtface, mf = mface; i < totface; mf++, tf++, i++, tangent += 4) {
-			len = (mf->v4) ? 4 : 3;
-
-			if (mtface == NULL) {
-				map_to_sphere(&uv[0][0], &uv[0][1], orco[mf->v1][0], orco[mf->v1][1], orco[mf->v1][2]);
-				map_to_sphere(&uv[1][0], &uv[1][1], orco[mf->v2][0], orco[mf->v2][1], orco[mf->v2][2]);
-				map_to_sphere(&uv[2][0], &uv[2][1], orco[mf->v3][0], orco[mf->v3][1], orco[mf->v3][2]);
-				if (len == 4)
-					map_to_sphere(&uv[3][0], &uv[3][1], orco[mf->v4][0], orco[mf->v4][1], orco[mf->v4][2]);
-			}
-		
-			mf_vi[0] = mf->v1;
-			mf_vi[1] = mf->v2;
-			mf_vi[2] = mf->v3;
-			mf_vi[3] = mf->v4;
-		
-			for (j = 0; j < len; j++) {
-				vtang = find_vertex_tangent(vtangents[mf_vi[j]], mtface ? tf->uv[j] : uv[j]);
-				normalize_v3_v3(tangent[j], vtang);
-				((float *) tangent[j])[3] = 1.0f;
-			}
-		}
-	}
-	
-	BLI_memarena_free(arena);
-	MEM_freeN(vtangents);
 }
 
 void DM_calc_auto_bump_scale(DerivedMesh *dm)
diff --git a/source/blender/blenlib/BLI_math_geom.h b/source/blender/blenlib/BLI_math_geom.h
index 107b688b36a..f781714e935 100644
--- a/source/blender/blenlib/BLI_math_geom.h
+++ b/source/blender/blenlib/BLI_math_geom.h
@@ -250,14 +250,6 @@ void accumulate_vertex_normals_poly(float **vertnos, float polyno[3],
 
 /********************************* Tangents **********************************/
 
-typedef struct VertexTangent {
-	struct VertexTangent *next;
-	float tang[3], uv[2];
-} VertexTangent;
-
-float *find_vertex_tangent(VertexTangent *vtang, const float uv[2]);
-void sum_or_add_vertex_tangent(void *arena, VertexTangent **vtang,
-                               const float tang[3], const float uv[2]);
 void tangent_from_uv(float uv1[2], float uv2[2], float uv3[2],
                      float co1[3], float co2[3], float co3[3], float n[3], float tang[3]);
 
diff --git a/source/blender/blenlib/intern/math_geom.c b/source/blender/blenlib/intern/math_geom.c
index 1d59f55a04b..3fcc75db5b6 100644
--- a/source/blender/blenlib/intern/math_geom.c
+++ b/source/blender/blenlib/intern/math_geom.c
@@ -2691,50 +2691,6 @@ void accumulate_vertex_normals_poly(float **vertnos, float polyno[3],
 
 /********************************* Tangents **********************************/
 
-/* For normal map tangents we need to detect uv boundaries, and only average
- * tangents in case the uvs are connected. Alternative would be to store 1
- * tangent per face rather than 4 per face vertex, but that's not compatible
- * with games */
-
-
-/* from BKE_mesh.h */
-#define STD_UV_CONNECT_LIMIT  0.0001f
-
-void sum_or_add_vertex_tangent(void *arena, VertexTangent **vtang, const float tang[3], const float uv[2])
-{
-	VertexTangent *vt;
-
-	/* find a tangent with connected uvs */
-	for (vt = *vtang; vt; vt = vt->next) {
-		if (fabsf(uv[0] - vt->uv[0]) < STD_UV_CONNECT_LIMIT && fabsf(uv[1] - vt->uv[1]) < STD_UV_CONNECT_LIMIT) {
-			add_v3_v3(vt->tang, tang);
-			return;
-		}
-	}
-
-	/* if not found, append a new one */
-	vt = BLI_memarena_alloc((MemArena *) arena, sizeof(VertexTangent));
-	copy_v3_v3(vt->tang, tang);
-	vt->uv[0] = uv[0];
-	vt->uv[1] = uv[1];
-
-	if (*vtang)
-		vt->next = *vtang;
-	*vtang = vt;
-}
-
-float *find_vertex_tangent(VertexTangent *vtang, const float uv[2])
-{
-	VertexTangent *vt;
-	static float nulltang[3] = {0.0f, 0.0f, 0.0f};
-
-	for (vt = vtang; vt; vt = vt->next)
-		if (fabsf(uv[0] - vt->uv[0]) < STD_UV_CONNECT_LIMIT && fabsf(uv[1] - vt->uv[1]) < STD_UV_CONNECT_LIMIT)
-			return vt->tang;
-
-	return nulltang; /* shouldn't happen, except for nan or so */
-}
-
 void tangent_from_uv(float uv1[2], float uv2[2], float uv3[3], float co1[3], float co2[3], float co3[3], float n[3], float tang[3])
 {
 	float s1 = uv2[0] - uv1[0];
diff --git a/source/blender/render/intern/source/convertblender.c b/source/blender/render/intern/source/convertblender.c
index 6034fd5c52f..c04bcce15ef 100644
--- a/source/blender/render/intern/source/convertblender.c
+++ b/source/blender/render/intern/source/convertblender.c
@@ -446,7 +446,7 @@ static void calc_edge_stress(Render *UNUSED(re), ObjectRen *obr, Mesh *me)
 }
 
 /* gets tangent from tface or orco */
-static void calc_tangent_vector(ObjectRen *obr, VertexTangent **vtangents, MemArena *arena, VlakRen *vlr, int do_nmap_tangent, int do_tangent)
+static void calc_tangent_vector(ObjectRen *obr, VlakRen *vlr, int do_tangent)
 {
 	MTFace *tface= RE_vlakren_get_tface(obr, vlr, obr->actmtface, NULL, 0);
 	VertRen *v1=vlr->v1, *v2=vlr->v2, *v3=vlr->v3, *v4=vlr->v4;
@@ -481,12 +481,6 @@ static void calc_tangent_vector(ObjectRen *obr, VertexTangent **vtangents, MemAr
 		add_v3_v3(tav, tang);
 	}
 	
-	if (do_nmap_tangent) {
-		sum_or_add_vertex_tangent(arena, &vtangents[v1->index], tang, uv1);
-		sum_or_add_vertex_tangent(arena, &vtangents[v2->index], tang, uv2);
-		sum_or_add_vertex_tangent(arena, &vtangents[v3->index], tang, uv3);
-	}
-
 	if (v4) {
 		tangent_from_uv(uv1, uv3, uv4, v1->co, v3->co, v4->co, vlr->n, tang);
 		
@@ -498,12 +492,6 @@ static void calc_tangent_vector(ObjectRen *obr, VertexTangent **vtangents, MemAr
 			tav= RE_vertren_get_tangent(obr, v4, 1);
 			add_v3_v3(tav, tang);
 		}
-
-		if (do_nmap_tangent) {
-			sum_or_add_vertex_tangent(arena, &vtangents[v1->index], tang, uv1);
-			sum_or_add_vertex_tangent(arena, &vtangents[v3->index], tang, uv3);
-			sum_or_add_vertex_tangent(arena, &vtangents[v4->index], tang, uv4);
-		}
 	}
 }
 
@@ -568,8 +556,14 @@ static void GetNormal(const SMikkTSpaceContext * pContext, float fNorm[], const
 	//assert(vert_index>=0 && vert_index<4);
 	SRenderMeshToTangent * pMesh = (SRenderMeshToTangent *) pContext->m_pUserData;
 	VlakRen *vlr= RE_findOrAddVlak(pMesh->obr, face_num);
-	const float *n= (&vlr->v1)[vert_index]->n;
-	copy_v3_v3(fNorm, n);
+
+	if(vlr->flag & ME_SMOOTH) {
+		const float *n= (&vlr->v1)[vert_index]->n;
+		copy_v3_v3(fNorm, n);
+	}
+	else {
+		negate_v3_v3(fNorm, vlr->n);
+	}
 }
 static void SetTSpace(const SMikkTSpaceContext * pContext, const float fvTangent[], const float fSign, const int face_num, const int iVert)
 {
@@ -585,17 +579,8 @@ static void SetTSpace(const SMikkTSpaceContext * pContext, const float fvTangent
 
 static void calc_vertexnormals(Render *UNUSED(re), ObjectRen *obr, int do_tangent, int do_nmap_tangent)
 {
-	MemArena *arena= NULL;
-	VertexTangent **vtangents= NULL;
 	int a;
 
-	if (do_nmap_tangent) {
-		arena= BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, "nmap tangent arena");
-		BLI_memarena_use_calloc(arena);
-
-		vtangents= MEM_callocN(sizeof(VertexTangent*)*obr->totvert, "VertexTangent");
-	}
-
 		/* clear all vertex normals */
 	for (a=0; a<obr->totvert; a++) {
 		VertRen *ver= RE_findOrAddVert(obr, a);
@@ -613,10 +598,10 @@ static void calc_vertexnormals(Render *UNUSED(re), ObjectRen *obr, int do_tangen
 			accumulate_vertex_normals(vlr->v1->n, vlr->v2->n, vlr->v3->n, n4,
 				vlr->n, vlr->v1->co, vlr->v2->co, vlr->v3->co, c4);
 		}
-		if (do_nmap_tangent || do_tangent) {
+		if (do_tangent) {
 			/* tangents still need to be calculated for flat faces too */
 			/* weighting removed, they are not vertexnormals */
-			calc_tangent_vector(obr, vtangents, arena, vlr, do_nmap_tangent, do_tangent);
+			calc_tangent_vector(obr, vlr, do_tangent);
 		}
 	}
 
@@ -630,32 +615,6 @@ static void calc_vertexnormals(Render *UNUSED(re), ObjectRen *obr, int do_tangen
 			if (is_zero_v3(vlr->v3->n)) copy_v3_v3(vlr->v3->n, vlr->n);
 			if (vlr->v4 && is_zero_v3(vlr->v4->n)) copy_v3_v3(vlr->v4->n, vlr->n);
 		}
-
-		if (do_nmap_tangent) {
-			VertRen *v1=vlr->v1, *v2=vlr->v2, *v3=vlr->v3, *v4=vlr->v4;
-			MTFace *tface= RE_vlakren_get_tface(obr, vlr, obr->actmtface, NULL, 0);
-
-			if (tface) {
-				int k=0;
-				float *vtang, *ftang= RE_vlakren_get_nmap_tangent(obr, vlr, 1);
-
-				vtang= find_vertex_tangent(vtangents[v1->index], tface->uv[0]);
-				copy_v3_v3(ftang, vtang);
-				normalize_v3(ftang);
-				vtang= find_vertex_tangent(vtangents[v2->index], tface->uv[1]);
-				copy_v3_v3(ftang+4, vtang);
-				normalize_v3(ftang+4);
-				vtang= find_vertex_tangent(vtangents[v3->index], tface->uv[2]);
-				copy_v3_v3(ftang+8, vtang);
-				normalize_v3(ftang+8);
-				if (v4) {
-					vtang= find_vertex_tangent(vtangents[v4->index], tface->uv[3]);
-					copy_v3_v3(ftang+12, vtang);
-					normalize_v3(ftang+12);
-				}
-				for (k=0; k<4; k++) ftang[4*k+3]=1;
-			}
-		}
 	}
 	
 	/* normalize vertex normals */
@@ -675,6 +634,7 @@ static void calc_vertexnormals(Render *UNUSED(re), ObjectRen *obr, int do_tangen
 		}
 	}
 
+	/* normal mapping tangent with mikktspace */
 	if (do_nmap_tangent != FALSE) {
 		SRenderMeshToTangent mesh2tangent;
 		SMikkTSpaceContext sContext;
@@ -696,11 +656,6 @@ static void calc_vertexnormals(Render *UNUSED(re), ObjectRen *obr, int do_tangen
 
 		genTangSpaceDefault(&sContext);
 	}
-
-	if (arena)
-		BLI_memarena_free(arena);
-	if (vtangents)
-		MEM_freeN(vtangents);
 }
 
 /* ------------------------------------------------------------------------- */
@@ -3274,11 +3229,11 @@ static void init_render_mesh(Render *re, ObjectRen *obr, int timeoffset)
 			/* normalmaps, test if tangents needed, separated from shading */
 			if (ma->mode_l & MA_TANGENT_V) {
 				need_tangent= 1;
-				if (me->mtface==NULL)
+				if (me->mtpoly==NULL)
 					need_orco= 1;
 			}
 			if (ma->mode_l & MA_NORMAP_TANG) {
-				if (me->mtface==NULL) {
+				if (me->mtpoly==NULL) {
 					need_orco= 1;
 					need_tangent= 1;
 				}
@@ -3289,7 +3244,7 @@ static void init_render_mesh(Render *re, ObjectRen *obr, int timeoffset)
 
 	if (re->flag & R_NEED_TANGENT) {
 		/* exception for tangent space baking */
-		if (me->mtface==NULL) {
+		if (me->mtpoly==NULL) {
 			need_orco= 1;
 			need_tangent= 1;
 		}