Fix [#30234] Various problems with CD layers and tesselation, related to modifiers stack.

Should also fix [#30266], [#29451], and partly [#30316].

Here are the changes made by this commit:
* It adds a "dirty" flag to DerivedMesh struct (for now, only DM_DIRTY_TESS_CDLAYERS, but more might be added as needed).
* It adds a new func, DM_update_tessface_data, which assumes tessfaces themselves are valid, but updates tessellated customdata from their poly/loop counter parts.
* At end of modstack, when valid tessellated faces are present in finaldm , but the cdlayers dirty flag is set, call that function (instead of recomputing the whole tessellation).
* Edits to the codes concerned (UVProject, DynamicPaint, and Subsurf modifiers).
* Also add to subsurf dm generation code the creation of a CD_POLYINDEX layer (mandatory for DM_update_tessface_data to work well, and imho all tessellated dm should have one).

Note: some pieces of old code are just #if 0’ed, will clean them later.

7 files changed, 128 insertions, 9 deletions

diff --git a/source/blender/blenkernel/BKE_DerivedMesh.h b/source/blender/blenkernel/BKE_DerivedMesh.h
index 493a5ec285f..aefa94d07de 100644
--- a/source/blender/blenkernel/BKE_DerivedMesh.h
+++ b/source/blender/blenkernel/BKE_DerivedMesh.h
@@ -147,6 +147,11 @@ typedef enum DMDrawFlag {
 	DM_DRAW_ALWAYS_SMOOTH = 2
 } DMDrawFlag;
 
+typedef enum DMDirtyFlag {
+	/* dm has valid tessellated faces, but tessellated CDDATA need to be updated. */
+	DM_DIRTY_TESS_CDLAYERS = 1 << 0,
+} DMDirtyFlag;
+
 typedef struct DerivedMesh DerivedMesh;
 struct DerivedMesh {
 	/* Private DerivedMesh data, only for internal DerivedMesh use */
@@ -158,6 +163,7 @@ struct DerivedMesh {
 	struct GPUDrawObject *drawObject;
 	DerivedMeshType type;
 	float auto_bump_scale;
+	DMDirtyFlag dirty;
 
 	/* calculate vert and face normals */
 	void (*calcNormals)(DerivedMesh *dm);
@@ -540,6 +546,8 @@ void DM_DupPolys(DerivedMesh *source, DerivedMesh *target);
 
 void DM_ensure_tessface(DerivedMesh *dm);
 
+void DM_update_tessface_data(DerivedMesh *dm);
+
 /* interpolates vertex data from the vertices indexed by src_indices in the
  * source mesh using the given weights and stores the result in the vertex
  * indexed by dest_index in the dest mesh
diff --git a/source/blender/blenkernel/BKE_mesh.h b/source/blender/blenkernel/BKE_mesh.h
index c5d137b086f..546fd2bb354 100644
--- a/source/blender/blenkernel/BKE_mesh.h
+++ b/source/blender/blenkernel/BKE_mesh.h
@@ -297,7 +297,7 @@ void BKE_mesh_tessface_calc(struct Mesh *mesh);
 void BKE_mesh_tessface_ensure(struct Mesh *mesh);
 void BKE_mesh_tessface_clear(struct Mesh *mesh);
 
-/*convert a triangle of loop facedata to mface facedata*/
+/* Convert a triangle or quadrangle of loop/poly data to tessface data */
 void mesh_loops_to_mface_corners(struct CustomData *fdata, struct CustomData *ldata,
                                  struct CustomData *pdata, int lindex[4], int findex,
                                  const int polyindex, const int mf_len,
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 906733d1f26..e533fa77135 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -67,6 +67,7 @@
 #include "BKE_tessmesh.h"
 #include "BKE_bvhutils.h"
 #include "BKE_deform.h"
+#include "BKE_global.h" /* For debug flag, DM_update_tessface_data() func. */
 
 #ifdef WITH_GAMEENGINE
 #include "BKE_navmesh_conversion.h"
@@ -286,6 +287,7 @@ void DM_init(DerivedMesh *dm, DerivedMeshType type, int numVerts, int numEdges,
 	
 	dm->needsFree = 1;
 	dm->auto_bump_scale = -1.0f;
+	dm->dirty = 0;
 }
 
 void DM_from_template(DerivedMesh *dm, DerivedMesh *source, DerivedMeshType type,
@@ -313,6 +315,7 @@ void DM_from_template(DerivedMesh *dm, DerivedMesh *source, DerivedMeshType type
 	DM_init_funcs(dm);
 
 	dm->needsFree = 1;
+	dm->dirty = 0;
 }
 
 int DM_release(DerivedMesh *dm)
@@ -381,6 +384,73 @@ void DM_ensure_tessface(DerivedMesh *dm)
 	}
 }
 
+/* Update tessface CD data from loop/poly ones. Needed when not retesselating after modstack evaluation. */
+/* NOTE: Assumes dm has valid tesselated data! */
+void DM_update_tessface_data(DerivedMesh *dm)
+{
+	MFace *mf = dm->getTessFaceArray(dm);
+	MPoly *mp = dm->getPolyArray(dm);
+	MLoop *ml = dm->getLoopArray(dm);
+
+	CustomData *fdata = dm->getTessFaceDataLayout(dm);
+	CustomData *pdata = dm->getPolyDataLayout(dm);
+	CustomData *ldata = dm->getLoopDataLayout(dm);
+
+	const int numTex = CustomData_number_of_layers(pdata, CD_MTEXPOLY);
+	const int numCol = CustomData_number_of_layers(ldata, CD_MLOOPCOL);
+	const int hasWCol = CustomData_has_layer(ldata, CD_WEIGHT_MLOOPCOL);
+	const int hasOrigSpace = CustomData_has_layer(ldata, CD_ORIGSPACE_MLOOP);
+
+	int *polyindex = CustomData_get_layer(fdata, CD_POLYINDEX);
+
+	int mf_idx,
+	    totface = dm->getNumTessFaces(dm),
+	    ml_idx[4];
+
+	/* Should never occure, but better abort than segfault! */
+	if (!polyindex)
+		return;
+
+	CustomData_from_bmeshpoly(fdata, pdata, ldata, totface);
+
+	for (mf_idx = 0; mf_idx < totface; mf_idx++, mf++) {
+		const int mf_len = mf->v4 ? 4 : 3;
+		int i, not_done;
+
+		/* Find out loop indices. */
+		/* XXX Is there a better way to do this? */
+		/* NOTE: This assumes tessface are valid and in sync with loop/poly… Else, most likely, segfault! */
+		for (i = mp[polyindex[mf_idx]].loopstart, not_done = mf_len; not_done; i++) {
+			MLoop *tml = &ml[i];
+			if (tml->v == mf->v1) {
+				ml_idx[0] = i;
+				not_done--;
+			}
+			else if (tml->v == mf->v2) {
+				ml_idx[1] = i;
+				not_done--;
+			}
+			else if (tml->v == mf->v3) {
+				ml_idx[2] = i;
+				not_done--;
+			}
+			else if (mf_len == 4 && tml->v == mf->v4) {
+				ml_idx[3] = i;
+				not_done--;
+			}
+		}
+		mesh_loops_to_mface_corners(fdata, ldata, pdata,
+		                            ml_idx, mf_idx, polyindex[mf_idx],
+		                            mf_len,
+		                            numTex, numCol, hasWCol, hasOrigSpace);
+	}
+
+	if (G.f & G_DEBUG)
+		printf("Updated tessellated customdata of dm %p\n", dm);
+
+	dm->dirty &= ~DM_DIRTY_TESS_CDLAYERS;
+}
+
 void DM_to_mesh(DerivedMesh *dm, Mesh *me, Object *ob)
 {
 	/* dm might depend on me, so we need to do everything with a local copy */
@@ -1062,7 +1132,9 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 #endif
 	MLoop *mloop = dm->getLoopArray(dm), *ml;
 	MPoly *mp = dm->getPolyArray(dm);
+#if 0
 	int numFaces = dm->getNumTessFaces(dm);
+#endif
 	int numVerts = dm->getNumVerts(dm);
 	int totloop;
 	int i, j;
@@ -1075,9 +1147,11 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 	if (wtcol_f) {
 		unsigned char *wtcol_f_step = wtcol_f;
 # else
+#if 0
 	/* XXX We have to create a CD_WEIGHT_MCOL, else it might sigsev (after a SubSurf mod, eg)... */
 	if(!dm->getTessFaceDataArray(dm, CD_WEIGHT_MCOL))
 		CustomData_add_layer(&dm->faceData, CD_WEIGHT_MCOL, CD_CALLOC, NULL, numFaces);
+#endif
 
 	{
 #endif
@@ -1162,6 +1236,8 @@ void DM_update_weight_mcol(Object *ob, DerivedMesh *dm, int const draw_flag,
 		}
 		MEM_freeN(wtcol_v);
 	}
+
+	dm->dirty |= DM_DIRTY_TESS_CDLAYERS;
 }
 
 
@@ -1274,7 +1350,8 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 	Mesh *me = ob->data;
 	ModifierData *firstmd, *md, *previewmd = NULL;
 	LinkNode *datamasks, *curr;
-	CustomDataMask mask, nextmask, append_mask = 0;
+	/* XXX Always copying POLYINDEX, else tesselated data are no more valid! */
+	CustomDataMask mask, nextmask, append_mask = CD_MASK_POLYINDEX;
 	float (*deformedVerts)[3] = NULL;
 	DerivedMesh *dm=NULL, *orcodm, *clothorcodm, *finaldm;
 	int numVerts = me->totvert;
@@ -1582,11 +1659,11 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 			/* in case of dynamic paint, make sure preview mask remains for following modifiers */
 			/* XXX Temp and hackish solution! */
 			if (md->type == eModifierType_DynamicPaint)
-				append_mask |= CD_MASK_WEIGHT_MCOL;
+				append_mask |= CD_MASK_WEIGHT_MLOOPCOL;
 			/* In case of active preview modifier, make sure preview mask remains for following modifiers. */
 			else if ((md == previewmd) && (do_mod_wmcol)) {
 				DM_update_weight_mcol(ob, dm, draw_flag, NULL, 0, NULL);
-				append_mask |= CD_MASK_WEIGHT_MCOL;
+				append_mask |= CD_MASK_WEIGHT_MLOOPCOL;
 			}
 		}
 
@@ -1674,7 +1751,9 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 
 	{
 		/* calculating normals can re-calculate tessfaces in some cases */
+#if 0
 		int num_tessface = finaldm->getNumTessFaces(finaldm);
+#endif
 		/* --------------------------------------------------------------------- */
 		/* First calculate the polygon and vertex normals, re-tessellation
 		 * copies these into the tessface's normal layer */
@@ -1693,14 +1772,25 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		 * redundant in cases where the render mode doesn't use these inputs, but
 		 * ideally eventually tessellation would happen on-demand, and this is one
 		 * of the primary places it would be needed. */
-		if (num_tessface == 0 && finaldm->getNumTessFaces(finaldm) == 0) {
+#if 0
+		if (num_tessface == 0 && finaldm->getNumTessFaces(finaldm) == 0)
+#else
+		if (finaldm->getNumTessFaces(finaldm) == 0) /* || !CustomData_has_layer(&finaldm->faceData, CD_POLYINDEX)) */
+#endif
+		{
 			finaldm->recalcTessellation(finaldm);
 		}
+		/* Even if tesselation is not needed, some modifiers migh have modified CD layers
+		 * (like mloopcol or mloopuv), hence we have to update those. */
+		else if (finaldm->dirty & DM_DIRTY_TESS_CDLAYERS) {
+			/* A tesselation already exists, it should always have a CD_POLYINDEX. */
+			BLI_assert(CustomData_has_layer(&finaldm->faceData, CD_POLYINDEX));
+			DM_update_tessface_data(finaldm);
+		}
 		/* Need to watch this, it can cause issues, see bug [#29338]             */
 		/* take care with this block, we really need testing frameworks          */
 		/* --------------------------------------------------------------------- */
 
-
 		/* without this, drawing ngon tri's faces will show ugly tessellated face
 		 * normals and will also have to calculate normals on the fly, try avoid
 		 * this where possible since calculating polygon normals isn't fast,
@@ -1714,7 +1804,6 @@ static void mesh_calc_modifiers(Scene *scene, Object *ob, float (*inputVertexCos
 		}
 	}
 
-
 	*final_r = finaldm;
 
 	if(orcodm)
diff --git a/source/blender/blenkernel/intern/cdderivedmesh.c b/source/blender/blenkernel/intern/cdderivedmesh.c
index 85620faf993..9aa052ef29f 100644
--- a/source/blender/blenkernel/intern/cdderivedmesh.c
+++ b/source/blender/blenkernel/intern/cdderivedmesh.c
@@ -1957,6 +1957,7 @@ static DerivedMesh *cddm_copy_ex(DerivedMesh *source, int faces_from_tessfaces)
 	DM_from_template(dm, source, DM_TYPE_CDDM, numVerts, numEdges, numTessFaces,
 		numLoops, numPolys);
 	dm->deformedOnly = source->deformedOnly;
+	dm->dirty = source->dirty;
 
 	CustomData_copy_data(&source->vertData, &dm->vertData, 0, 0, numVerts);
 	CustomData_copy_data(&source->edgeData, &dm->edgeData, 0, 0, numEdges);
diff --git a/source/blender/blenkernel/intern/dynamicpaint.c b/source/blender/blenkernel/intern/dynamicpaint.c
index 310d370de38..56637dc84e1 100644
--- a/source/blender/blenkernel/intern/dynamicpaint.c
+++ b/source/blender/blenkernel/intern/dynamicpaint.c
@@ -1612,12 +1612,14 @@ static struct DerivedMesh *dynamicPaint_Modifier_apply(DynamicPaintModifierData
 							MPoly *mp = CDDM_get_polys(result);
 							int totpoly = result->numPolyData;
 
+#if 0
 							/* XXX We have to create a CD_WEIGHT_MCOL, else it might sigsev
 							 *     (after a SubSurf mod, eg)... */
 							if(!result->getTessFaceDataArray(result, CD_WEIGHT_MCOL)) {
 								int numFaces = result->getNumTessFaces(result);
 								CustomData_add_layer(&result->faceData, CD_WEIGHT_MCOL, CD_CALLOC, NULL, numFaces);
 							}
+#endif
 
 							/* Save preview results to weight layer to be
 							*   able to share same drawing methods */
@@ -1702,6 +1704,9 @@ static struct DerivedMesh *dynamicPaint_Modifier_apply(DynamicPaintModifierData
 								col[i].a = 255;
 							}
 						}
+
+						/* Mark tessellated CD layers as dirty. */
+						result->dirty |= DM_DIRTY_TESS_CDLAYERS;
 					}
 					/* vertex group paint */
 					else if (surface->type == MOD_DPAINT_SURFACE_T_WEIGHT) {
@@ -1712,7 +1717,8 @@ static struct DerivedMesh *dynamicPaint_Modifier_apply(DynamicPaintModifierData
 						/* viewport preview */
 						if (surface->flags & MOD_DPAINT_PREVIEW) {
 							/* Save preview results to weight layer to be
-							*   able to share same drawing methods */
+							 * able to share same drawing methods.
+							 * Note this func also sets DM_DIRTY_TESS_CDLAYERS flag! */
 							DM_update_weight_mcol(ob, result, 0, weight, 0, NULL);
 						}
 
diff --git a/source/blender/blenkernel/intern/subsurf_ccg.c b/source/blender/blenkernel/intern/subsurf_ccg.c
index b968afa4d22..7fcb866d806 100644
--- a/source/blender/blenkernel/intern/subsurf_ccg.c
+++ b/source/blender/blenkernel/intern/subsurf_ccg.c
@@ -2888,7 +2888,7 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 	int *vertOrigIndex, *faceOrigIndex, *polyOrigIndex, *base_polyOrigIndex; /* *edgeOrigIndex - as yet, unused  */
 	short *edgeFlags;
 	DMFlagMat *faceFlags;
-	int *loopidx = NULL, *vertidx = NULL;
+	int *loopidx = NULL, *vertidx = NULL, *polyidx = NULL;
 	BLI_array_declare(loopidx);
 	BLI_array_declare(vertidx);
 	int loopindex, loopindex2;
@@ -2922,6 +2922,7 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 	if (
 	        (numTex && CustomData_number_of_layers(&ccgdm->dm.faceData, CD_MTFACE) != numTex)  ||
 	        (numCol && CustomData_number_of_layers(&ccgdm->dm.faceData, CD_MCOL) != numCol)    ||
+	        (hasWCol && !CustomData_has_layer(&ccgdm->dm.faceData, CD_WEIGHT_MCOL))            ||
 	        (hasOrigSpace && !CustomData_has_layer(&ccgdm->dm.faceData, CD_ORIGSPACE)) )
 	{
 		CustomData_from_bmeshpoly(&ccgdm->dm.faceData,
@@ -2930,6 +2931,10 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 		                          ccgSubSurf_getNumFinalFaces(ss));
 	}
 
+	/* We absolutely need that layer, else it's no valid tesselated data! */
+	polyidx = CustomData_add_layer(&ccgdm->dm.faceData, CD_POLYINDEX, CD_CALLOC,
+	                               NULL, ccgSubSurf_getNumFinalFaces(ss));
+
 	ccgdm->dm.getMinMax = ccgDM_getMinMax;
 	ccgdm->dm.getNumVerts = ccgDM_getNumVerts;
 	ccgdm->dm.getNumEdges = ccgDM_getNumEdges;
@@ -3207,6 +3212,9 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 
 					ccgdm->reverseFaceMap[faceNum] = index;
 
+					/* This is a simple one to one mapping, here... */
+					polyidx[faceNum] = faceNum;
+
 					faceNum++;
 				}
 			}
@@ -3297,6 +3305,9 @@ static CCGDerivedMesh *getCCGDerivedMesh(CCGSubSurf *ss,
 	ccgdm->dm.numLoopData = loopindex2;
 	ccgdm->dm.numPolyData = faceNum;
 
+	/* All tessellated CD layers were updated! */
+	ccgdm->dm.dirty &= ~DM_DIRTY_TESS_CDLAYERS;
+
 	BLI_array_free(vertidx);
 	BLI_array_free(loopidx);
 	free_ss_weights(&wtable);
diff --git a/source/blender/modifiers/intern/MOD_uvproject.c b/source/blender/modifiers/intern/MOD_uvproject.c
index cd4858fcb4d..b9a4821bb7a 100644
--- a/source/blender/modifiers/intern/MOD_uvproject.c
+++ b/source/blender/modifiers/intern/MOD_uvproject.c
@@ -378,6 +378,10 @@ static DerivedMesh *uvprojectModifier_do(UVProjectModifierData *umd,
 			}
 		}
 	}
+
+	/* Mark tessellated CD layers as dirty. */
+	dm->dirty |= DM_DIRTY_TESS_CDLAYERS;
+
 	return dm;
 }