Fix modifiers stack not recalculated when mapping requirements change

Reported by pixaal on irc, basically reproducable by inserting bevel
modifier on cube and entering/exiting texture paint mode.

Now object stores last needsMapping variable as well as customdata mask.

Also now texture painting only needs mapping when we are in texture
paint selection mode, so modifiers that don't support mapping can still
be used to paint now.

2 files changed, 29 insertions, 19 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 411de80ddbc..d850dd3ee26 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -2292,13 +2292,8 @@ static void editbmesh_calc_modifiers(Scene *scene, Object *ob, BMEditMesh *em, D
 }
 
 static void mesh_build_data(Scene *scene, Object *ob, CustomDataMask dataMask,
-                            int build_shapekey_layers)
+                            int build_shapekey_layers, int needMapping)
 {
-	Object *obact = scene->basact ? scene->basact->object : NULL;
-	bool editing = BKE_paint_select_face_test(ob);
-	/* weight paint and face select need original indices because of selection buffer drawing */
-	int needMapping = (ob == obact) && (editing || (ob->mode & (OB_MODE_WEIGHT_PAINT | OB_MODE_VERTEX_PAINT | OB_MODE_TEXTURE_PAINT)));
-
 	BLI_assert(ob->type == OB_MESH);
 
 	BKE_object_free_derived_caches(ob);
@@ -2313,6 +2308,7 @@ static void mesh_build_data(Scene *scene, Object *ob, CustomDataMask dataMask,
 	ob->derivedFinal->needsFree = 0;
 	ob->derivedDeform->needsFree = 0;
 	ob->lastDataMask = dataMask;
+	ob->lastNeedMapping = needMapping;
 
 	if ((ob->mode & OB_MODE_SCULPT) && ob->sculpt) {
 		/* create PBVH immediately (would be created on the fly too,
@@ -2341,14 +2337,23 @@ static void editbmesh_build_data(Scene *scene, Object *obedit, BMEditMesh *em, C
 	BLI_assert(!(em->derivedFinal->dirty & DM_DIRTY_NORMALS));
 }
 
-static CustomDataMask object_get_datamask(const Scene *scene, Object *ob)
+static CustomDataMask object_get_datamask(const Scene *scene, Object *ob, int *r_needmapping)
 {
 	Object *actob = scene->basact ? scene->basact->object : NULL;
 	CustomDataMask mask = ob->customdata_mask;
+	bool editing = BKE_paint_select_face_test(ob);
+
+	if (r_needmapping)
+		*r_needmapping = 0;
 
 	if (ob == actob) {
+
+		/* weight paint and face select need original indices because of selection buffer drawing */
+		if (r_needmapping)
+			*r_needmapping = (editing || (ob->mode & (OB_MODE_WEIGHT_PAINT | OB_MODE_VERTEX_PAINT)));
+
 		/* check if we need tfaces & mcols due to face select or texture paint */
-		if ((ob->mode & OB_MODE_TEXTURE_PAINT) || BKE_paint_select_face_test(ob)) {
+		if ((ob->mode & OB_MODE_TEXTURE_PAINT) || editing) {
 			mask |= CD_MASK_MTFACE | CD_MASK_MCOL;
 		}
 
@@ -2371,13 +2376,14 @@ static CustomDataMask object_get_datamask(const Scene *scene, Object *ob)
 void makeDerivedMesh(Scene *scene, Object *ob, BMEditMesh *em,
                      CustomDataMask dataMask, int build_shapekey_layers)
 {
-	dataMask |= object_get_datamask(scene, ob);
+	int needMapping;
+	dataMask |= object_get_datamask(scene, ob, &needMapping);
 
 	if (em) {
 		editbmesh_build_data(scene, ob, em, dataMask);
 	}
 	else {
-		mesh_build_data(scene, ob, dataMask, build_shapekey_layers);
+		mesh_build_data(scene, ob, dataMask, build_shapekey_layers, needMapping);
 	}
 }
 
@@ -2388,10 +2394,11 @@ DerivedMesh *mesh_get_derived_final(Scene *scene, Object *ob, CustomDataMask dat
 	/* if there's no derived mesh or the last data mask used doesn't include
 	 * the data we need, rebuild the derived mesh
 	 */
-	dataMask |= object_get_datamask(scene, ob);
+	int needMapping;
+	dataMask |= object_get_datamask(scene, ob, &needMapping);
 
-	if (!ob->derivedFinal || (dataMask & ob->lastDataMask) != dataMask)
-		mesh_build_data(scene, ob, dataMask, 0);
+	if (!ob->derivedFinal || (dataMask & ob->lastDataMask) != dataMask || (needMapping != ob->lastNeedMapping))
+		mesh_build_data(scene, ob, dataMask, 0, needMapping);
 
 	if (ob->derivedFinal) { BLI_assert(!(ob->derivedFinal->dirty & DM_DIRTY_NORMALS)); }
 	return ob->derivedFinal;
@@ -2402,10 +2409,12 @@ DerivedMesh *mesh_get_derived_deform(Scene *scene, Object *ob, CustomDataMask da
 	/* if there's no derived mesh or the last data mask used doesn't include
 	 * the data we need, rebuild the derived mesh
 	 */
-	dataMask |= object_get_datamask(scene, ob);
+	int needmapping;
+
+	dataMask |= object_get_datamask(scene, ob, &needmapping);
 
-	if (!ob->derivedDeform || (dataMask & ob->lastDataMask) != dataMask)
-		mesh_build_data(scene, ob, dataMask, 0);
+	if (!ob->derivedDeform || (dataMask & ob->lastDataMask) != dataMask || (needmapping != ob->lastNeedMapping))
+		mesh_build_data(scene, ob, dataMask, 0, needmapping);
 
 	return ob->derivedDeform;
 }
@@ -2494,7 +2503,7 @@ DerivedMesh *editbmesh_get_derived_cage_and_final(Scene *scene, Object *obedit,
 	/* if there's no derived mesh or the last data mask used doesn't include
 	 * the data we need, rebuild the derived mesh
 	 */
-	dataMask |= object_get_datamask(scene, obedit);
+	dataMask |= object_get_datamask(scene, obedit, NULL);
 
 	if (!em->derivedCage ||
 	    (em->lastDataMask & dataMask) != dataMask)
@@ -2512,7 +2521,7 @@ DerivedMesh *editbmesh_get_derived_cage(Scene *scene, Object *obedit, BMEditMesh
 	/* if there's no derived mesh or the last data mask used doesn't include
 	 * the data we need, rebuild the derived mesh
 	 */
-	dataMask |= object_get_datamask(scene, obedit);
+	dataMask |= object_get_datamask(scene, obedit, NULL);
 
 	if (!em->derivedCage ||
 	    (em->lastDataMask & dataMask) != dataMask)
diff --git a/source/blender/makesdna/DNA_object_types.h b/source/blender/makesdna/DNA_object_types.h
index 8bd676caa5e..50f94aa09e5 100644
--- a/source/blender/makesdna/DNA_object_types.h
+++ b/source/blender/makesdna/DNA_object_types.h
@@ -191,7 +191,8 @@ typedef struct Object {
 	/* dupli-frame settings */
 	int dupon, dupoff, dupsta, dupend;
 
-	int pad;
+	/* did last modifier stack generation need mapping support? */
+	int lastNeedMapping;
 
 	/* during realtime */