Move tangents and baking to looptris:

Notes:

* Code in rendering and in game engine will still convert
tangents to a tessface representation. Added code that
takes care of tangent layer only, might be removed
when BGE and rendering goes full mlooptri mode.

* Baking should work discovered some dead code while
I was working on the patch, also tangents are broken
when baking from multires (also in master), but those
are separate issues that can be fixed later.

This should fix T45491 as well

1 files changed, 121 insertions, 59 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 0b1b4310ca4..3156eb391b8 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -521,7 +521,8 @@ void DM_update_tessface_data(DerivedMesh *dm)
 	    CustomData_has_layer(fdata, CD_MCOL) ||
 	    CustomData_has_layer(fdata, CD_PREVIEW_MCOL) ||
 	    CustomData_has_layer(fdata, CD_ORIGSPACE) ||
-	    CustomData_has_layer(fdata, CD_TESSLOOPNORMAL))
+	    CustomData_has_layer(fdata, CD_TESSLOOPNORMAL) ||
+	    CustomData_has_layer(fdata, CD_TANGENT))
 	{
 		loopindex = MEM_mallocN(sizeof(*loopindex) * totface, __func__);
 
@@ -557,6 +558,69 @@ void DM_update_tessface_data(DerivedMesh *dm)
 	dm->dirty &= ~DM_DIRTY_TESS_CDLAYERS;
 }
 
+void DM_generate_tangent_tessface_data(DerivedMesh *dm, bool generate)
+{
+	int i;
+	MFace *mf, *mface = dm->getTessFaceArray(dm);
+	MPoly *mp = dm->getPolyArray(dm);
+	MLoop *ml = dm->getLoopArray(dm);
+
+	CustomData *fdata = dm->getTessFaceDataLayout(dm);
+	CustomData *pdata = dm->getPolyDataLayout(dm);
+	CustomData *ldata = dm->getLoopDataLayout(dm);
+
+	const int totface = dm->getNumTessFaces(dm);
+	int mf_idx;
+
+	int *polyindex = CustomData_get_layer(fdata, CD_ORIGINDEX);
+	unsigned int (*loopindex)[4];
+
+	/* Should never occure, but better abort than segfault! */
+	if (!polyindex)
+		return;
+
+	CustomData_from_bmeshpoly(fdata, pdata, ldata, totface);
+
+	if (generate) {
+		for (i = 0; i < ldata->totlayer; i++) {
+			if (ldata->layers[i].type == CD_TANGENT)
+				CustomData_add_layer_named(fdata, CD_TANGENT, CD_CALLOC, NULL, totface, ldata->layers[i].name);
+		}
+		CustomData_bmesh_update_active_layers(fdata, pdata, ldata);
+	}
+
+	loopindex = MEM_mallocN(sizeof(*loopindex) * totface, __func__);
+
+	for (mf_idx = 0, mf = mface; mf_idx < totface; mf_idx++, mf++) {
+		const int mf_len = mf->v4 ? 4 : 3;
+		unsigned int *ml_idx = loopindex[mf_idx];
+		int i, not_done;
+
+		/* Find out loop indices. */
+		/* NOTE: This assumes tessface are valid and in sync with loop/poly... Else, most likely, segfault! */
+		for (i = mp[polyindex[mf_idx]].loopstart, not_done = mf_len; not_done; i++) {
+			const int tf_v = BKE_MESH_TESSFACE_VINDEX_ORDER(mf, ml[i].v);
+			if (tf_v != -1) {
+				ml_idx[tf_v] = i;
+				not_done--;
+			}
+		}
+	}
+
+	/* NOTE: quad detection issue - forth vertidx vs forth loopidx:
+	 * Here, our tfaces' forth vertex index is never 0 for a quad. However, we know our forth loop index may be
+	 * 0 for quads (because our quads may have been rotated compared to their org poly, see tessellation code).
+	 * So we pass the MFace's, and BKE_mesh_loops_to_tessdata will use MFace->v4 index as quad test.
+	 */
+	BKE_mesh_tangent_loops_to_tessdata(fdata, ldata, mface, polyindex, loopindex, totface);
+
+	MEM_freeN(loopindex);
+
+	if (G.debug & G_DEBUG)
+		printf("%s: Updated tessellated tangents of dm %p\n", __func__, dm);
+}
+
+
 void DM_update_materials(DerivedMesh *dm, Object *ob)
 {
 	int i, totmat = ob->totcol + 1; /* materials start from 1, default material is 0 */
@@ -2899,9 +2963,11 @@ void mesh_get_mapped_verts_coords(DerivedMesh *dm, float (*r_cos)[3], const int
 
 typedef struct {
 	float (*precomputedFaceNormals)[3];
-	short (*precomputedLoopNormals)[4][3];
-	MTFace *mtface;     /* texture coordinates */
-	MFace *mface;       /* indices */
+	float (*precomputedLoopNormals)[3];
+	const MLoopTri *looptri;
+	MLoopUV *mloopuv;   /* texture coordinates */
+	MPoly *mpoly;       /* indices */
+	MLoop *mloop;       /* indices */
 	MVert *mvert;       /* vertices & normals */
 	float (*orco)[3];
 	float (*tangent)[4];    /* destination */
@@ -2920,15 +2986,17 @@ static int GetNumFaces(const SMikkTSpaceContext *pContext)
 
 static int GetNumVertsOfFace(const SMikkTSpaceContext *pContext, const int face_num)
 {
-	SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
-	return pMesh->mface[face_num].v4 != 0 ? 4 : 3;
+	//SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
+	UNUSED_VARS(pContext, face_num);
+	return 3;
 }
 
 static void GetPosition(const SMikkTSpaceContext *pContext, float r_co[3], const int face_num, const int vert_index)
 {
 	//assert(vert_index >= 0 && vert_index < 4);
 	SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
-	const float *co = pMesh->mvert[(&pMesh->mface[face_num].v1)[vert_index]].co;
+	const MLoopTri *lt = &pMesh->looptri[face_num];
+	const float *co = pMesh->mvert[pMesh->mloop[lt->tri[vert_index]].v].co;
 	copy_v3_v3(r_co, co);
 }
 
@@ -2936,13 +3004,14 @@ static void GetTextureCoordinate(const SMikkTSpaceContext *pContext, float r_uv[
 {
 	//assert(vert_index >= 0 && vert_index < 4);
 	SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
+	const MLoopTri *lt = &pMesh->looptri[face_num];
 
-	if (pMesh->mtface != NULL) {
-		const float *uv = pMesh->mtface[face_num].uv[vert_index];
+	if (pMesh->mloopuv != NULL) {
+		const float *uv = pMesh->mloopuv[lt->tri[vert_index]].uv;
 		copy_v2_v2(r_uv, uv);
 	}
 	else {
-		const float *orco = pMesh->orco[(&pMesh->mface[face_num].v1)[vert_index]];
+		const float *orco = pMesh->orco[pMesh->mloop[lt->tri[vert_index]].v];
 		map_to_sphere(&r_uv[0], &r_uv[1], orco[0], orco[1], orco[2]);
 	}
 }
@@ -2951,41 +3020,36 @@ static void GetNormal(const SMikkTSpaceContext *pContext, float r_no[3], const i
 {
 	//assert(vert_index >= 0 && vert_index < 4);
 	SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
-	const bool smoothnormal = (pMesh->mface[face_num].flag & ME_SMOOTH) != 0;
+	const MLoopTri *lt = &pMesh->looptri[face_num];
+	const bool smoothnormal = (pMesh->mpoly[lt->poly].flag & ME_SMOOTH) != 0;
 
 	if (pMesh->precomputedLoopNormals) {
-		normal_short_to_float_v3(r_no, pMesh->precomputedLoopNormals[face_num][vert_index]);
+		copy_v3_v3(r_no, pMesh->precomputedLoopNormals[lt->tri[vert_index]]);
 	}
 	else if (!smoothnormal) {    // flat
 		if (pMesh->precomputedFaceNormals) {
-			copy_v3_v3(r_no, pMesh->precomputedFaceNormals[face_num]);
+			copy_v3_v3(r_no, pMesh->precomputedFaceNormals[lt->poly]);
 		}
 		else {
-			MFace *mf = &pMesh->mface[face_num];
-			const float *p0 = pMesh->mvert[mf->v1].co;
-			const float *p1 = pMesh->mvert[mf->v2].co;
-			const float *p2 = pMesh->mvert[mf->v3].co;
-
-			if (mf->v4) {
-				const float *p3 = pMesh->mvert[mf->v4].co;
-				normal_quad_v3(r_no, p0, p1, p2, p3);
-			}
-			else {
-				normal_tri_v3(r_no, p0, p1, p2);
-			}
+			const float *p0 = pMesh->mvert[pMesh->mloop[lt->tri[0]].v].co;
+			const float *p1 = pMesh->mvert[pMesh->mloop[lt->tri[1]].v].co;
+			const float *p2 = pMesh->mvert[pMesh->mloop[lt->tri[2]].v].co;
+
+			normal_tri_v3(r_no, p0, p1, p2);
 		}
 	}
 	else {
-		const short *no = pMesh->mvert[(&pMesh->mface[face_num].v1)[vert_index]].no;
+		const short *no = pMesh->mvert[pMesh->mloop[lt->tri[0]].v].no;
 		normal_short_to_float_v3(r_no, no);
 	}
 }
 
-static void SetTSpace(const SMikkTSpaceContext *pContext, const float fvTangent[3], const float fSign, const int face_num, const int iVert)
+static void SetTSpace(const SMikkTSpaceContext *pContext, const float fvTangent[3], const float fSign, const int face_num, const int vert_index)
 {
 	//assert(vert_index >= 0 && vert_index < 4);
 	SGLSLMeshToTangent *pMesh = (SGLSLMeshToTangent *) pContext->m_pUserData;
-	float *pRes = pMesh->tangent[4 * face_num + iVert];
+	const MLoopTri *lt = &pMesh->looptri[face_num];
+	float *pRes = pMesh->tangent[lt->tri[vert_index]];
 	copy_v3_v3(pRes, fvTangent);
 	pRes[3] = fSign;
 }
@@ -2993,40 +3057,44 @@ static void SetTSpace(const SMikkTSpaceContext *pContext, const float fvTangent[
 void DM_add_tangent_layer(DerivedMesh *dm)
 {
 	/* mesh vars */
+	const MLoopTri *looptri;
 	MVert *mvert;
-	MTFace *mtface;
-	MFace *mface;
+	MLoopUV *mloopuv;
+	MPoly *mpoly;
+	MLoop *mloop;
 	float (*orco)[3] = NULL, (*tangent)[4];
 	int /* totvert, */ totface;
 	float (*fnors)[3];
-	short (*tlnors)[4][3];
+	float (*tlnors)[3];
 
-	if (CustomData_get_layer_index(&dm->faceData, CD_TANGENT) != -1)
+	if (CustomData_get_layer_index(&dm->loopData, CD_TANGENT) != -1)
 		return;
 
-	fnors = dm->getTessFaceDataArray(dm, CD_NORMAL);
+	fnors = dm->getPolyDataArray(dm, CD_NORMAL);
 	/* Note, we assume we do have tessellated loop normals at this point (in case it is object-enabled),
 	 * have to check this is valid...
 	 */
-	tlnors = dm->getTessFaceDataArray(dm, CD_TESSLOOPNORMAL);
+	tlnors = dm->getLoopDataArray(dm, CD_NORMAL);
 
 	/* check we have all the needed layers */
 	/* totvert = dm->getNumVerts(dm); */ /* UNUSED */
-	totface = dm->getNumTessFaces(dm);
+	looptri = dm->getLoopTriArray(dm);
+	totface = dm->getNumLoopTri(dm);
 
 	mvert = dm->getVertArray(dm);
-	mface = dm->getTessFaceArray(dm);
-	mtface = dm->getTessFaceDataArray(dm, CD_MTFACE);
+	mpoly = dm->getPolyArray(dm);
+	mloop = dm->getLoopArray(dm);
+	mloopuv = dm->getLoopDataArray(dm, CD_MLOOPUV);
 
-	if (!mtface) {
+	if (!mloopuv) {
 		orco = dm->getVertDataArray(dm, CD_ORCO);
 		if (!orco)
 			return;
 	}
 	
 	/* create tangent layer */
-	DM_add_tessface_layer(dm, CD_TANGENT, CD_CALLOC, NULL);
-	tangent = DM_get_tessface_data_layer(dm, CD_TANGENT);
+	DM_add_loop_layer(dm, CD_TANGENT, CD_CALLOC, NULL);
+	tangent = DM_get_loop_data_layer(dm, CD_TANGENT);
 	
 	/* new computation method */
 	{
@@ -3036,8 +3104,10 @@ void DM_add_tangent_layer(DerivedMesh *dm)
 
 		mesh2tangent.precomputedFaceNormals = fnors;
 		mesh2tangent.precomputedLoopNormals = tlnors;
-		mesh2tangent.mtface = mtface;
-		mesh2tangent.mface = mface;
+		mesh2tangent.looptri = looptri;
+		mesh2tangent.mloopuv = mloopuv;
+		mesh2tangent.mpoly = mpoly;
+		mesh2tangent.mloop = mloop;
 		mesh2tangent.mvert = mvert;
 		mesh2tangent.orco = orco;
 		mesh2tangent.tangent = tangent;
@@ -3208,7 +3278,7 @@ void DM_calc_auto_bump_scale(DerivedMesh *dm)
 
 void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs, DMVertexAttribs *attribs)
 {
-	CustomData *vdata, *fdata, *tfdata = NULL;
+	CustomData *vdata, *ldata;
 	int a, b, layer;
 
 	/* From the layers requested by the GLSL shader, figure out which ones are
@@ -3217,7 +3287,7 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 	memset(attribs, 0, sizeof(DMVertexAttribs));
 
 	vdata = &dm->vertData;
-	fdata = tfdata = dm->getTessFaceDataLayout(dm);
+	ldata = dm->getLoopDataLayout(dm);
 	
 	/* calc auto bump scale if necessary */
 	if (dm->auto_bump_scale <= 0.0f)
@@ -3226,7 +3296,7 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 	/* add a tangent layer if necessary */
 	for (b = 0; b < gattribs->totlayer; b++) {
 		if (gattribs->layer[b].type == CD_TANGENT) {
-			if (CustomData_get_layer_index(fdata, CD_TANGENT) == -1) {
+			if (CustomData_get_layer_index(ldata, CD_TANGENT) == -1) {
 				DM_add_tangent_layer(dm);
 				break;
 			}
@@ -3261,9 +3331,6 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 				attribs->tface[a].gl_texco = gattribs->layer[b].gltexco;
 			}
 			else {
-				/* exception .. */
-				CustomData *ldata = dm->getLoopDataLayout(dm);
-
 				if (gattribs->layer[b].name[0])
 					layer = CustomData_get_named_layer_index(ldata, CD_MLOOPUV,
 					                                         gattribs->layer[b].name);
@@ -3287,9 +3354,6 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 		}
 		else if (gattribs->layer[b].type == CD_MCOL) {
 			if (dm->type == DM_TYPE_EDITBMESH) {
-				/* exception .. */
-				CustomData *ldata = dm->getLoopDataLayout(dm);
-
 				if (gattribs->layer[b].name[0])
 					layer = CustomData_get_named_layer_index(ldata, CD_MLOOPCOL,
 					                                         gattribs->layer[b].name);
@@ -3311,9 +3375,6 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 				attribs->mcol[a].gl_index = gattribs->layer[b].glindex;
 			}
 			else {
-				/* exception .. */
-				CustomData *ldata = dm->getLoopDataLayout(dm);
-
 				/* vertex colors */
 				if (gattribs->layer[b].name[0])
 					layer = CustomData_get_named_layer_index(ldata, CD_MLOOPCOL,
@@ -3337,14 +3398,13 @@ void DM_vertex_attributes_from_gpu(DerivedMesh *dm, GPUVertexAttribs *gattribs,
 			}
 		}
 		else if (gattribs->layer[b].type == CD_TANGENT) {
-			/* tangents */
-			layer = CustomData_get_layer_index(fdata, CD_TANGENT);
+			layer = CustomData_get_layer_index(ldata, CD_TANGENT);
 
 			attribs->tottang = 1;
 
 			if (layer != -1) {
-				attribs->tang.array = fdata->layers[layer].data;
-				attribs->tang.em_offset = fdata->layers[layer].offset;
+				attribs->tang.array = ldata->layers[layer].data;
+				attribs->tang.em_offset = ldata->layers[layer].offset;
 			}
 			else {
 				attribs->tang.array = NULL;
@@ -3386,6 +3446,8 @@ void DM_draw_attrib_vertex(DMVertexAttribs *attribs, int a, int index, int vert,
 	const float zero[4] = {0.0f, 0.0f, 0.0f, 0.0f};
 	int b;
 
+	UNUSED_VARS(a, vert);
+
 	/* orco texture coordinates */
 	if (attribs->totorco) {
 		/*const*/ float (*array)[3] = attribs->orco.array;
@@ -3433,7 +3495,7 @@ void DM_draw_attrib_vertex(DMVertexAttribs *attribs, int a, int index, int vert,
 	/* tangent for normal mapping */
 	if (attribs->tottang) {
 		/*const*/ float (*array)[4] = attribs->tang.array;
-		const float *tang = (array) ? array[a * 4 + vert] : zero;
+		const float *tang = (array) ? array[loop] : zero;
 		glVertexAttrib4fvARB(attribs->tang.gl_index, tang);
 	}
 }