Fix [#30234] Various problems with CD layers and tesselation, related to modifiers stack.

Should also fix [#30266], [#29451], and partly [#30316].

Here are the changes made by this commit:
* It adds a "dirty" flag to DerivedMesh struct (for now, only DM_DIRTY_TESS_CDLAYERS, but more might be added as needed).
* It adds a new func, DM_update_tessface_data, which assumes tessfaces themselves are valid, but updates tessellated customdata from their poly/loop counter parts.
* At end of modstack, when valid tessellated faces are present in finaldm , but the cdlayers dirty flag is set, call that function (instead of recomputing the whole tessellation).
* Edits to the codes concerned (UVProject, DynamicPaint, and Subsurf modifiers).
* Also add to subsurf dm generation code the creation of a CD_POLYINDEX layer (mandatory for DM_update_tessface_data to work well, and imho all tessellated dm should have one).

Note: some pieces of old code are just #if 0’ed, will clean them later.

1 files changed, 95 insertions, 6 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 906733d1f26..e533fa77135 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -67,6 +67,7 @@
 #include "BKE_tessmesh.h"
 #include "BKE_bvhutils.h"
 #include "BKE_deform.h"
+#include "BKE_global.h" /* For debug flag, DM_update_tessface_data() func. */
 
 #ifdef WITH_GAMEENGINE
 #include "BKE_navmesh_conversion.h"
@@ -286,6 +287,7 @@ void DM_init(DerivedMesh *dm, DerivedMeshType type, int numVerts, int numEdges,
 	
 	dm->needsFree = 1;
 	dm->auto_bump_scale = -1.0f;
+	dm->dirty = 0;
 }
 
 void DM_from_template(DerivedMesh *dm, DerivedMesh *source, DerivedMeshType type,
@@ -313,6 +315,7 @@ void DM_from_template(DerivedMesh *dm, DerivedMesh *source, DerivedMeshType type
 	DM_init_funcs(dm);
 
 	dm->needsFree = 1;
+	dm->dirty = 0;
 }
 
 int DM_release(DerivedMesh *dm)
@@ -381,6 +384,73 @@ void DM_ensure_tessface(DerivedMesh *dm)
 	}
 }
 
+/* Update tessface CD data from loop/poly ones. Needed when not retesselating after modstack evaluation. */
+/* NOTE: Assumes dm has valid tesselated data! */
+void DM_update_tessface_data(DerivedMesh *dm)
+{
+	MFace *mf = dm->getTessFaceArray(dm);
+	MPoly *mp = dm->getPolyArray(dm);
+	MLoop *ml = dm->getLoopArray(dm);
+
+	CustomData *fdata = dm->getTessFaceDataLayout(dm);
+	CustomData *pdata = dm->getPolyDataLayout(dm);
+	CustomData *ldata = dm->getLoopDataLayout(dm);
+
+	const int numTex = CustomData_number_of_layers(pdata, CD_MTEXPOLY);
+	const int numCol = CustomData_number_of_layers(ldata, CD_MLOOPCOL);
+	const int hasWCol = CustomData_has_layer(ldata, CD_WEIGHT_MLOOPCOL);
+	const int hasOrigSpace = CustomData_has_layer(ldata, CD_ORIGSPACE_MLOOP);
+
+	int *polyindex = CustomData_get_layer(fdata, CD_POLYINDEX);
+
+	int mf_idx,
+	    totface = dm->getNumTessFaces(dm),
+	    ml_idx[4];
+
+	/* Should never occure, but better abort than segfault! */
+	if (!polyindex)
+		return;
+
+	CustomData_from_bmeshpoly(fdata, pdata, ldata, totface);
+
+	for (mf_idx = 0; mf_idx < totface; mf_idx++, mf++) {
+		const int mf_len = mf->v4 ? 4 : 3;
+		int i, not_done;
+
+		/* Find out loop indices. */
+		/* XXX Is there a better way to do this? */
+		/* NOTE: This assumes tessface are valid and in sync with loop/poly… Else, most likely, segfault! */
+		for (i = mp[polyindex[mf_idx]].loopstart, not_done = mf_len; not_done; i++) {
+			MLoop *tml = &ml[i];
+			if (tml->v == mf->v1) {
+				ml_idx[0] = i;
+				not_done--;
+			}
+			else if (tml->v == mf->v2) {
+				ml_idx[1] = i;
+				not_done--;
+			}
+			else if (tml->v == mf->v3) {
+				ml_idx[2] = i;
+				not_done--;
+			}
+			else if (mf_len == 4 && tml->v == mf->v4) {
+				ml_idx[3] = i;
+				not_done--;
+			}
+		}
+		mesh_loops_to_mface_corners(fdata, ldata, pdata,
+		                            ml_idx, mf_idx, polyindex[mf_idx],
+		                            mf_len,
+		                            numTex, numCol, hasWCol, hasOrigSpace);
+	}
+
+	if (G.f & G_DEBUG)
+		printf("Updated tessellated customdata of dm %p\n", dm);
+
+	dm->dirty &= ~DM_DIRTY_TESS_CDLAYERS;
+}
+
 void DM_to_mesh(DerivedMesh *dm, Mesh *me, Object *ob)
 {
 	/* dm might depend on me, so we need to do everything with a local copy */
@@ -1062,7 +1132,9 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 #endif
 	MLoop *mloop = dm->getLoopArray(dm), *ml;
 	MPoly *mp = dm->getPolyArray(dm);
+#if 0
 	int numFaces = dm->getNumTessFaces(dm);
+#endif
 	int numVerts = dm->getNumVerts(dm);
 	int totloop;
 	int i, j;
@@ -1075,9 +1147,11 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 	if (wtcol_f) {
 		unsigned char *wtcol_f_step = wtcol_f;
 # else
+#if 0
 	/* XXX We have to create a CD_WEIGHT_MCOL, else it might sigsev (after a SubSurf mod, eg)... */
 	if(!dm->getTessFaceDataArray(dm, CD_WEIGHT_MCOL))
 		CustomData_add_layer(&dm->faceData, CD_WEIGHT_MCOL, CD_CALLOC, NULL, numFaces);
+#endif
 
 	{
 #endif
@@ -1162,6 +1236,8 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 		}
 		MEM_freeN(wtcol_v);
 	}
+
+	dm->dirty |= DM_DIRTY_TESS_CDLAYERS;
 }
 
 
@@ -1274,7 +1350,8 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 	Mesh *me = ob->data;
 	ModifierData *firstmd, *md, *previewmd = NULL;
 	LinkNode *datamasks, *curr;
-	CustomDataMask mask, nextmask, append_mask = 0;
+	/* XXX Always copying POLYINDEX, else tesselated data are no more valid! */
+	CustomDataMask mask, nextmask, append_mask = CD_MASK_POLYINDEX;
 	float (*deformedVerts)[3] = NULL;
 	DerivedMesh *dm=NULL, *orcodm, *clothorcodm, *finaldm;
 	int numVerts = me->totvert;
@@ -1582,11 +1659,11 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 			/* in case of dynamic paint, make sure preview mask remains for following modifiers */
 			/* XXX Temp and hackish solution! */
 			if (md->type == eModifierType_DynamicPaint)
-				append_mask |= CD_MASK_WEIGHT_MCOL;
+				append_mask |= CD_MASK_WEIGHT_MLOOPCOL;
 			/* In case of active preview modifier, make sure preview mask remains for following modifiers. */
 			else if ((md == previewmd) && (do_mod_wmcol)) {
 				DM_update_weight_mcol(ob, dm, draw_flag, NULL, 0, NULL);
-				append_mask |= CD_MASK_WEIGHT_MCOL;
+				append_mask |= CD_MASK_WEIGHT_MLOOPCOL;
 			}
 		}
 
@@ -1674,7 +1751,9 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 
 	{
 		/* calculating normals can re-calculate tessfaces in some cases */
+#if 0
 		int num_tessface = finaldm->getNumTessFaces(finaldm);
+#endif
 		/* --------------------------------------------------------------------- */
 		/* First calculate the polygon and vertex normals, re-tessellation
 		 * copies these into the tessface's normal layer */
@@ -1693,14 +1772,25 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		 * redundant in cases where the render mode doesn't use these inputs, but
 		 * ideally eventually tessellation would happen on-demand, and this is one
 		 * of the primary places it would be needed. */
-		if (num_tessface == 0 && finaldm->getNumTessFaces(finaldm) == 0) {
+#if 0
+		if (num_tessface == 0 && finaldm->getNumTessFaces(finaldm) == 0)
+#else
+		if (finaldm->getNumTessFaces(finaldm) == 0) /* || !CustomData_has_layer(&finaldm->faceData, CD_POLYINDEX)) */
+#endif
+		{
 			finaldm->recalcTessellation(finaldm);
 		}
+		/* Even if tesselation is not needed, some modifiers migh have modified CD layers
+		 * (like mloopcol or mloopuv), hence we have to update those. */
+		else if (finaldm->dirty & DM_DIRTY_TESS_CDLAYERS) {
+			/* A tesselation already exists, it should always have a CD_POLYINDEX. */
+			BLI_assert(CustomData_has_layer(&finaldm->faceData, CD_POLYINDEX));
+			DM_update_tessface_data(finaldm);
+		}
 		/* Need to watch this, it can cause issues, see bug [#29338]             */
 		/* take care with this block, we really need testing frameworks          */
 		/* --------------------------------------------------------------------- */
 
-
 		/* without this, drawing ngon tri's faces will show ugly tessellated face
 		 * normals and will also have to calculate normals on the fly, try avoid
 		 * this where possible since calculating polygon normals isn't fast,
@@ -1714,7 +1804,6 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		}
 	}
 
-
 	*final_r = finaldm;
 
 	if(orcodm)