Fix #27930: many modifiers crashed when used on a lattice with a vertex group.

11 files changed, 49 insertions, 72 deletions

diff --git a/source/blender/modifiers/intern/MOD_cast.c b/source/blender/modifiers/intern/MOD_cast.c
index 5cb352ef482..14b23ba4972 100644
--- a/source/blender/modifiers/intern/MOD_cast.c
+++ b/source/blender/modifiers/intern/MOD_cast.c
@@ -177,10 +177,7 @@ static void sphere_do(
 
 	/* 3) if we were given a vertex group name,
 	* only those vertices should be affected */
-	defgrp_index = defgroup_name_index(ob, cmd->defgrp_name);
-
-	if ((ob->type == OB_MESH) && dm && defgrp_index >= 0)
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, cmd->defgrp_name, &dvert, &defgrp_index);
 
 	if(flag & MOD_CAST_SIZE_FROM_RADIUS) {
 		len = cmd->radius;
@@ -335,10 +332,7 @@ static void cuboid_do(
 
 	/* 3) if we were given a vertex group name,
 	* only those vertices should be affected */
-	defgrp_index = defgroup_name_index(ob, cmd->defgrp_name);
-
-	if ((ob->type == OB_MESH) && dm && defgrp_index >= 0)
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, cmd->defgrp_name, &dvert, &defgrp_index);
 
 	if (ctrl_ob) {
 		if(flag & MOD_CAST_USE_OB_TRANSFORM) {
diff --git a/source/blender/modifiers/intern/MOD_displace.c b/source/blender/modifiers/intern/MOD_displace.c
index 01f1b6fb2a7..e0482e6b3fc 100644
--- a/source/blender/modifiers/intern/MOD_displace.c
+++ b/source/blender/modifiers/intern/MOD_displace.c
@@ -169,7 +169,7 @@ static void displaceModifier_do(
 {
 	int i;
 	MVert *mvert;
-	MDeformVert *dvert = NULL;
+	MDeformVert *dvert;
 	int defgrp_index;
 	float (*tex_co)[3];
 	float weight= 1.0f; /* init value unused but some compilers may complain */
@@ -177,11 +177,8 @@ static void displaceModifier_do(
 	if(!dmd->texture) return;
 	if(dmd->strength == 0.0f) return;
 
-	defgrp_index = defgroup_name_index(ob, dmd->defgrp_name);
-
 	mvert = CDDM_get_verts(dm);
-	if(defgrp_index >= 0)
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, dmd->defgrp_name, &dvert, &defgrp_index);
 
 	tex_co = MEM_callocN(sizeof(*tex_co) * numVerts,
 				 "displaceModifier_do tex_co");
diff --git a/source/blender/modifiers/intern/MOD_hook.c b/source/blender/modifiers/intern/MOD_hook.c
index 082c199b16f..ea8d602dd7a 100644
--- a/source/blender/modifiers/intern/MOD_hook.c
+++ b/source/blender/modifiers/intern/MOD_hook.c
@@ -157,9 +157,8 @@ static void deformVerts(ModifierData *md, Object *ob,
 	int i, *index_pt;
 	const float falloff_squared= hmd->falloff * hmd->falloff; /* for faster comparisons */
 	
-	int max_dvert= 0;
-	MDeformVert *dvert= NULL;
-	int defgrp_index = -1;
+	MDeformVert *dvert;
+	int defgrp_index, max_dvert;
 	
 	/* get world-space matrix of target, corrected for the space the verts are in */
 	if (hmd->subtarget[0] && pchan) {
@@ -174,21 +173,8 @@ static void deformVerts(ModifierData *md, Object *ob,
 	mul_serie_m4(mat, ob->imat, dmat, hmd->parentinv,
 			 NULL, NULL, NULL, NULL, NULL);
 
-	if((defgrp_index= defgroup_name_index(ob, hmd->name)) != -1) {
-		Mesh *me = ob->data;
-		if(dm) {
-			dvert= dm->getVertDataArray(dm, CD_MDEFORMVERT);
-			if(dvert) {
-				max_dvert = numVerts;
-			}
-		}
-		else if(me->dvert) {
-			dvert= me->dvert;
-			if(dvert) {
-				max_dvert = me->totvert;
-			}
-		}
-	}
+	modifier_get_vgroup(ob, dm, hmd->name, &dvert, &defgrp_index);
+	max_dvert = (dvert)? numVerts: 0;
 
 	/* Regarding index range checking below.
 	 *
diff --git a/source/blender/modifiers/intern/MOD_meshdeform.c b/source/blender/modifiers/intern/MOD_meshdeform.c
index 5021f3a6d2e..3903f2602e4 100644
--- a/source/blender/modifiers/intern/MOD_meshdeform.c
+++ b/source/blender/modifiers/intern/MOD_meshdeform.c
@@ -284,10 +284,7 @@ static void meshdeformModifier_do(
 			copy_v3_v3(dco[a], co);
 	}
 
-	defgrp_index = defgroup_name_index(ob, mmd->defgrp_name);
-
-	if(dm && defgrp_index >= 0)
-		dvert= dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, mmd->defgrp_name, &dvert, &defgrp_index);
 
 	/* do deformation */
 	fac= 1.0f;
diff --git a/source/blender/modifiers/intern/MOD_simpledeform.c b/source/blender/modifiers/intern/MOD_simpledeform.c
index ea4771b679a..5efd6cd28ec 100644
--- a/source/blender/modifiers/intern/MOD_simpledeform.c
+++ b/source/blender/modifiers/intern/MOD_simpledeform.c
@@ -162,8 +162,8 @@ static void SimpleDeformModifier_do(SimpleDeformModifierData *smd, struct Object
 	float smd_limit[2], smd_factor;
 	SpaceTransform *transf = NULL, tmp_transf;
 	void (*simpleDeform_callback)(const float factor, const float dcut[3], float *co) = NULL;	//Mode callback
-	int vgroup = defgroup_name_index(ob, smd->vgroup_name);
-	MDeformVert *dvert = NULL;
+	int vgroup;
+	MDeformVert *dvert;
 
 	//Safe-check
 	if(smd->origin == ob) smd->origin = NULL;					//No self references
@@ -216,17 +216,7 @@ static void SimpleDeformModifier_do(SimpleDeformModifierData *smd, struct Object
 		smd_factor   = smd->factor / MAX2(FLT_EPSILON, smd_limit[1]-smd_limit[0]);
 	}
 
-
-	if(dm)
-	{
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
-	}
-	else if(ob->type == OB_LATTICE)
-	{
-		dvert = lattice_get_deform_verts(ob);
-	}
-
-
+	modifier_get_vgroup(ob, dm, smd->vgroup_name, &dvert, &vgroup);
 
 	switch(smd->mode)
 	{
diff --git a/source/blender/modifiers/intern/MOD_smooth.c b/source/blender/modifiers/intern/MOD_smooth.c
index 5f76fad14b1..28a31b84ea5 100644
--- a/source/blender/modifiers/intern/MOD_smooth.c
+++ b/source/blender/modifiers/intern/MOD_smooth.c
@@ -123,10 +123,7 @@ static void smoothModifier_do(
 	medges = dm->getEdgeArray(dm);
 	numDMEdges = dm->getNumEdges(dm);
 
-	defgrp_index = defgroup_name_index(ob, smd->defgrp_name);
-
-	if (defgrp_index >= 0)
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, smd->defgrp_name, &dvert, &defgrp_index);
 
 	/* NOTICE: this can be optimized a little bit by moving the
 	* if (dvert) out of the loop, if needed */
diff --git a/source/blender/modifiers/intern/MOD_solidify.c b/source/blender/modifiers/intern/MOD_solidify.c
index 1b7b724835c..390a780e9e6 100644
--- a/source/blender/modifiers/intern/MOD_solidify.c
+++ b/source/blender/modifiers/intern/MOD_solidify.c
@@ -48,6 +48,7 @@
 
 
 #include "MOD_modifiertypes.h"
+#include "MOD_util.h"
 
 #include "MEM_guardedalloc.h"
 
@@ -235,12 +236,11 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 	float const ofs_new= smd->offset	- (((-smd->offset_fac + 1.0f) * 0.5f) * smd->offset);
 
 	/* weights */
-	MDeformVert *dvert= NULL, *dv= NULL;
+	MDeformVert *dvert, *dv= NULL;
 	const int defgrp_invert = ((smd->flag & MOD_SOLIDIFY_VGROUP_INV) != 0);
-	const int defgrp_index= defgroup_name_index(ob, smd->defgrp_name);
+	int defgrp_index;
 
-	if (defgrp_index >= 0)
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	modifier_get_vgroup(ob, dm, smd->defgrp_name, &dvert, &defgrp_index);
 
 	orig_mface = dm->getFaceArray(dm);
 	orig_medge = dm->getEdgeArray(dm);
diff --git a/source/blender/modifiers/intern/MOD_util.c b/source/blender/modifiers/intern/MOD_util.c
index 9fe37e2d174..e9b835eab81 100644
--- a/source/blender/modifiers/intern/MOD_util.c
+++ b/source/blender/modifiers/intern/MOD_util.c
@@ -37,6 +37,7 @@
 
 #include <string.h>
 
+#include "DNA_lattice_types.h"
 #include "DNA_modifier_types.h"
 #include "DNA_object_types.h"
 #include "DNA_curve_types.h"
@@ -47,6 +48,8 @@
 #include "BLI_math_matrix.h"
 
 #include "BKE_cdderivedmesh.h"
+#include "BKE_deform.h"
+#include "BKE_lattice.h"
 #include "BKE_mesh.h"
 #include "BKE_displist.h"
 
@@ -239,6 +242,19 @@ DerivedMesh *get_dm(Object *ob, struct EditMesh *em, DerivedMesh *dm, float (*ve
 	return dm;
 }
 
+void modifier_get_vgroup(Object *ob, DerivedMesh *dm, const char *name, MDeformVert **dvert, int *defgrp_index)
+{
+	*defgrp_index = defgroup_name_index(ob, name);
+	*dvert = NULL;
+
+	if(*defgrp_index >= 0) {
+		if(ob->type == OB_LATTICE)
+			*dvert = lattice_get_deform_verts(ob);
+		else if(dm)
+			*dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
+	}
+}
+
 /* only called by BKE_modifier.h/modifier.c */
 void modifier_type_init(ModifierTypeInfo *types[])
 {
diff --git a/source/blender/modifiers/intern/MOD_util.h b/source/blender/modifiers/intern/MOD_util.h
index b7862403459..b9b5c8a064a 100644
--- a/source/blender/modifiers/intern/MOD_util.h
+++ b/source/blender/modifiers/intern/MOD_util.h
@@ -36,14 +36,15 @@
 /* so modifier types match their defines */
 #include "MOD_modifiertypes.h"
 
-struct Tex;
-struct TexResult;
 struct CustomData;
 struct DerivedMesh;
-struct Object;
-struct Scene;
 struct EditMesh;
+struct MDeformVert;
 struct ModifierData;
+struct Object;
+struct Scene;
+struct Tex;
+struct TexResult;
 
 void get_texture_value(struct Tex *texture, float *tex_co, struct TexResult *texres);
 void get_texture_coords(struct MappingInfoModifierData *dmd, struct Object *ob, struct DerivedMesh *dm, float (*co)[3], float (*texco)[3], int numVerts);
@@ -51,5 +52,6 @@ void modifier_vgroup_cache(struct ModifierData *md, float (*vertexCos)[3]);
 void validate_layer_name(const struct CustomData *data, int type, char *name, char *outname);
 struct DerivedMesh *get_cddm(struct Object *ob, struct EditMesh *em, struct DerivedMesh *dm, float (*vertexCos)[3]);
 struct DerivedMesh *get_dm(struct Object *ob, struct EditMesh *em, struct DerivedMesh *dm, float (*vertexCos)[3], int orco);
+void modifier_get_vgroup(struct Object *ob, DerivedMesh *dm, const char *name, struct MDeformVert **dvert, int *defgrp_index);
 
 #endif /* MOD_UTIL_H */
diff --git a/source/blender/modifiers/intern/MOD_warp.c b/source/blender/modifiers/intern/MOD_warp.c
index 27add27deb1..2c77b486263 100644
--- a/source/blender/modifiers/intern/MOD_warp.c
+++ b/source/blender/modifiers/intern/MOD_warp.c
@@ -85,8 +85,8 @@ static CustomDataMask requiredDataMask(Object *UNUSED(ob), ModifierData *md)
 	CustomDataMask dataMask = 0;
 
 	/* ask for vertexgroups if we need them */
-	if(wmd->defgrp_name[0]) dataMask |= (1 << CD_MDEFORMVERT);
-	dataMask |= (1 << CD_MDEFORMVERT);
+	if(wmd->defgrp_name[0]) dataMask |= (CD_MASK_MDEFORMVERT);
+	dataMask |= (CD_MASK_MDEFORMVERT);
 
 	/* ask for UV coordinates if we need them */
 	if(wmd->texmapping == MOD_DISP_MAP_UV) dataMask |= (1 << CD_MTFACE);
@@ -174,14 +174,16 @@ static void warpModifier_do(WarpModifierData *wmd, Object *ob,
 	float strength = wmd->strength;
 	float fac = 1.0f, weight;
 	int i;
-	int defgrp_index = defgroup_name_index(ob, wmd->defgrp_name);
-	MDeformVert *dv= NULL;
+	int defgrp_index;
+	MDeformVert *dvert, *dv= NULL;
 
 	float (*tex_co)[3]= NULL;
 
 	if(!(wmd->object_from && wmd->object_to))
 		return;
 
+	modifier_get_vgroup(ob, dm, wmd->defgrp_name, &dvert, &defgrp_index);
+
 	if(wmd->curfalloff==NULL) /* should never happen, but bad lib linking could cause it */
 		wmd->curfalloff = curvemapping_add(1, 0.0f, 0.0f, 1.0f, 1.0f);
 
@@ -221,8 +223,8 @@ static void warpModifier_do(WarpModifierData *wmd, Object *ob,
 		        ((fac=len_v3v3(co, mat_from[3])) < wmd->falloff_radius && (fac=(wmd->falloff_radius-fac)/wmd->falloff_radius)) ) {
 
 			/* skip if no vert group found */
-			if(defgrp_index >= 0) {
-				dv = dm->getVertData(dm, i, CD_MDEFORMVERT);
+			if(dvert && defgrp_index >= 0) {
+				dv = &dvert[i];
 
 				if(dv) {
 					weight = defvert_find_weight(dv, defgrp_index) * wmd->strength;
diff --git a/source/blender/modifiers/intern/MOD_wave.c b/source/blender/modifiers/intern/MOD_wave.c
index 6dfe5314131..ca8161fe364 100644
--- a/source/blender/modifiers/intern/MOD_wave.c
+++ b/source/blender/modifiers/intern/MOD_wave.c
@@ -256,7 +256,7 @@ static void waveModifier_do(WaveModifierData *md,
 {
 	WaveModifierData *wmd = (WaveModifierData*) md;
 	MVert *mvert = NULL;
-	MDeformVert *dvert = NULL;
+	MDeformVert *dvert;
 	int defgrp_index;
 	float ctime = BKE_curframe(scene);
 	float minfac =
@@ -281,11 +281,7 @@ static void waveModifier_do(WaveModifierData *md,
 	}
 
 	/* get the index of the deform group */
-	defgrp_index = defgroup_name_index(ob, wmd->defgrp_name);
-
-	if(defgrp_index >= 0){
-		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
-	}
+	modifier_get_vgroup(ob, dm, wmd->defgrp_name, &dvert, &defgrp_index);
 
 	if(wmd->damp == 0) wmd->damp = 10.0f;