Fix T37914: Mask modifier behaves differently until painted

1 files changed, 21 insertions, 41 deletions

diff --git a/source/blender/modifiers/intern/MOD_mask.c b/source/blender/modifiers/intern/MOD_mask.c
index e754ac8f766..1d5dc29c119 100644
--- a/source/blender/modifiers/intern/MOD_mask.c
+++ b/source/blender/modifiers/intern/MOD_mask.c
@@ -96,14 +96,15 @@ static void updateDepgraph(ModifierData *md, DagForest *forest,
 }
 
 static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
-                                  DerivedMesh *derivedData,
+                                  DerivedMesh *dm,
                                   ModifierApplyFlag UNUSED(flag))
 {
 	MaskModifierData *mmd = (MaskModifierData *)md;
-	DerivedMesh *dm = derivedData, *result = NULL;
+	const bool found_test = (mmd->flag & MOD_MASK_INV) == 0;
+	DerivedMesh *result = NULL;
 	GHash *vertHash = NULL, *edgeHash, *polyHash;
 	GHashIterator *hashIter;
-	MDeformVert *dvert = NULL, *dv;
+	MDeformVert *dvert, *dv;
 	int numPolys = 0, numLoops = 0, numEdges = 0, numVerts = 0;
 	int maxVerts, maxEdges, maxPolys;
 	int i;
@@ -119,6 +120,11 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 
 	int *loop_mapping;
 
+	dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	if (dvert == NULL) {
+		return found_test ? CDDM_from_template(dm, 0, 0, 0, 0, 0) : dm;
+	}
+
 	/* Overview of Method:
 	 *	1. Get the vertices that are in the vertexgroup of interest 
 	 *	2. Filter out unwanted geometry (i.e. not in vertexgroup), by populating mappings with new vs old indices
@@ -136,7 +142,7 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 	if (!(ELEM(mmd->mode, MOD_MASK_MODE_ARM, MOD_MASK_MODE_VGROUP)) ||
 	    (maxVerts == 0) || (ob->defbase.first == NULL) )
 	{
-		return derivedData;
+		return dm;
 	}
 	
 	/* if mode is to use selected armature bones, aggregate the bone groups */
@@ -150,7 +156,7 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 		
 		/* check that there is armature object with bones to use, otherwise return original mesh */
 		if (ELEM3(NULL, oba, oba->pose, ob->defbase.first))
-			return derivedData;
+			return dm;
 		
 		/* determine whether each vertexgroup is associated with a selected bone or not 
 		 * - each cell is a boolean saying whether bone corresponding to the ith group is selected
@@ -168,16 +174,7 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 				bone_select_array[i] = FALSE;
 			}
 		}
-		
-		/* if no dverts (i.e. no data for vertex groups exists), we've got an
-		 * inconsistent situation, so free hashes and return original mesh
-		 */
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
-		if (dvert == NULL) {
-			MEM_freeN(bone_select_array);
-			return derivedData;
-		}
-		
+
 		/* verthash gives mapping from original vertex indices to the new indices (including selected matches only)
 		 * key = oldindex, value = newindex
 		 */
@@ -189,7 +186,7 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 		 */
 		for (i = 0, dv = dvert; i < maxVerts; i++, dv++) {
 			MDeformWeight *dw = dv->dw;
-			short found = 0;
+			bool found = false;
 			int j;
 			
 			/* check the groups that vertex is assigned to, and see if it was any use */
@@ -204,14 +201,8 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 				}
 			}
 			
-			/* check if include vert in vertHash */
-			if (mmd->flag & MOD_MASK_INV) {
-				/* if this vert is in the vgroup, don't include it in vertHash */
-				if (found) continue;
-			}
-			else {
-				/* if this vert isn't in the vgroup, don't include it in vertHash */
-				if (!found) continue;
+			if (found_test != found) {
+				continue;
 			}
 			
 			/* add to ghash for verts (numVerts acts as counter for mapping) */
@@ -224,13 +215,9 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 	}
 	else {  /* --- Using Nominated VertexGroup only --- */
 		int defgrp_index = defgroup_name_index(ob, mmd->vgroup);
-		
-		/* get dverts */
-		if (defgrp_index != -1)
-			dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
-			
+
 		/* if no vgroup (i.e. dverts) found, return the initial mesh */
-		if ((defgrp_index == -1) || (dvert == NULL))
+		if (defgrp_index == -1)
 			return dm;
 			
 		/* hashes for quickly providing a mapping from old to new - use key=oldindex, value=newindex */
@@ -238,18 +225,11 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 		
 		/* add vertices which exist in vertexgroup into ghash for filtering */
 		for (i = 0, dv = dvert; i < maxVerts; i++, dv++) {
-			const int weight_set = defvert_find_weight(dv, defgrp_index) != 0.0f;
-			
-			/* check if include vert in vertHash */
-			if (mmd->flag & MOD_MASK_INV) {
-				/* if this vert is in the vgroup, don't include it in vertHash */
-				if (weight_set) continue;
+			const bool found = defvert_find_weight(dv, defgrp_index) != 0.0f;
+			if (found_test != found) {
+				continue;
 			}
-			else {
-				/* if this vert isn't in the vgroup, don't include it in vertHash */
-				if (!weight_set) continue;
-			}
-			
+
 			/* add to ghash for verts (numVerts acts as counter for mapping) */
 			BLI_ghash_insert(vertHash, SET_INT_IN_POINTER(i), SET_INT_IN_POINTER(numVerts));
 			numVerts++;