Rigidbody: Add option to choose mesh source for collision shapes

The options are:
Base: Base mesh
Deform: shape keys and deform modifiers
Final: All deformations and modifiers

It would be nice to have a way of specifying where exactly in the
modifier stack the collision shape is generated. However this is not
staight forward since the rigid body simulation is not part of the
modifier system and would require hacks to make it work.

4 files changed, 64 insertions, 9 deletions

diff --git a/release/scripts/startup/bl_ui/properties_physics_rigidbody.py b/release/scripts/startup/bl_ui/properties_physics_rigidbody.py
index cd701178976..7a6c8e0a066 100644
--- a/release/scripts/startup/bl_ui/properties_physics_rigidbody.py
+++ b/release/scripts/startup/bl_ui/properties_physics_rigidbody.py
@@ -70,6 +70,9 @@ class PHYSICS_PT_rigid_body_collisions(PHYSICS_PT_rigidbody_panel, Panel):
 
         layout.prop(rbo, "collision_shape", text="Shape")
 
+        if rbo.collision_shape in {'MESH', 'CONVEX_HULL'}:
+            layout.prop(rbo, "mesh_source", text="Source")
+
         split = layout.split()
 
         col = split.column()
diff --git a/source/blender/blenkernel/intern/rigidbody.c b/source/blender/blenkernel/intern/rigidbody.c
index 1a7782f4b89..fa455fad090 100644
--- a/source/blender/blenkernel/intern/rigidbody.c
+++ b/source/blender/blenkernel/intern/rigidbody.c
@@ -225,26 +225,47 @@ void BKE_rigidbody_relink_constraint(RigidBodyCon *rbc)
 /* ************************************** */
 /* Setup Utilities - Validate Sim Instances */
 
+/* get the appropriate DerivedMesh based on rigid body mesh source */
+static DerivedMesh *rigidbody_get_mesh(Object *ob)
+{
+	if (ob->rigidbody_object->mesh_source == RBO_MESH_DEFORM) {
+		return ob->derivedDeform;
+	}
+	else if (ob->rigidbody_object->mesh_source == RBO_MESH_FINAL) {
+		return ob->derivedFinal;
+	}
+	else {
+		return CDDM_from_mesh(ob->data);
+	}
+}
+
 /* create collision shape of mesh - convex hull */
 static rbCollisionShape *rigidbody_get_shape_convexhull_from_mesh(Object *ob, float margin, bool *can_embed)
 {
 	rbCollisionShape *shape = NULL;
-	Mesh *me = NULL;
+	DerivedMesh *dm = NULL;
+	MVert *mvert = NULL;
+	int totvert = 0;
 
 	if (ob->type == OB_MESH && ob->data) {
-		me = ob->data;
+		dm = rigidbody_get_mesh(ob);
+		mvert   = (dm) ? dm->getVertArray(dm) : NULL;
+		totvert = (dm) ? dm->getNumVerts(dm) : 0;
 	}
 	else {
 		printf("ERROR: cannot make Convex Hull collision shape for non-Mesh object\n");
 	}
 
-	if (me && me->totvert) {
-		shape = RB_shape_new_convex_hull((float *)me->mvert, sizeof(MVert), me->totvert, margin, can_embed);
+	if (totvert) {
+		shape = RB_shape_new_convex_hull((float *)mvert, sizeof(MVert), totvert, margin, can_embed);
 	}
 	else {
 		printf("ERROR: no vertices to define Convex Hull collision shape with\n");
 	}
 
+	if (dm && ob->rigidbody_object->mesh_source == RBO_MESH_BASE)
+		dm->release(dm);
+
 	return shape;
 }
 
@@ -256,14 +277,18 @@ static rbCollisionShape *rigidbody_get_shape_trimesh_from_mesh(Object *ob)
 	rbCollisionShape *shape = NULL;
 
 	if (ob->type == OB_MESH) {
-		DerivedMesh *dm = CDDM_from_mesh(ob->data);
-
+		DerivedMesh *dm = NULL;
 		MVert *mvert;
 		MFace *mface;
 		int totvert;
 		int totface;
 
+		dm = rigidbody_get_mesh(ob);
+
 		/* ensure mesh validity, then grab data */
+		if (dm == NULL)
+			return NULL;
+
 		DM_ensure_tessface(dm);
 
 		mvert   = (dm) ? dm->getVertArray(dm) : NULL;
@@ -323,7 +348,7 @@ static rbCollisionShape *rigidbody_get_shape_trimesh_from_mesh(Object *ob)
 		}
 
 		/* cleanup temp data */
-		if (dm) {
+		if (dm && ob->rigidbody_object->mesh_source == RBO_MESH_BASE) {
 			dm->release(dm);
 		}
 	}
@@ -425,7 +450,8 @@ void BKE_rigidbody_validate_sim_shape(Object *ob, short rebuild)
 		rbo->physics_shape = new_shape;
 		RB_shape_set_margin(rbo->physics_shape, RBO_GET_MARGIN(rbo));
 	}
-	else { /* otherwise fall back to box shape */
+	/* use box shape if we can't fall back to old shape */
+	else if (rbo->physics_shape == NULL) {
 		rbo->shape = RB_SHAPE_BOX;
 		BKE_rigidbody_validate_sim_shape(ob, true);
 	}
@@ -798,6 +824,8 @@ RigidBodyOb *BKE_rigidbody_create_object(Scene *scene, Object *ob, short type)
 	else
 		rbo->shape = RB_SHAPE_TRIMESH;
 
+	rbo->mesh_source = RBO_MESH_DEFORM;
+
 	/* set initial transform */
 	mat4_to_loc_quat(rbo->pos, rbo->orn, ob->obmat);
 
diff --git a/source/blender/makesdna/DNA_rigidbody_types.h b/source/blender/makesdna/DNA_rigidbody_types.h
index 4a96c324f04..de23a3c2370 100644
--- a/source/blender/makesdna/DNA_rigidbody_types.h
+++ b/source/blender/makesdna/DNA_rigidbody_types.h
@@ -103,7 +103,8 @@ typedef struct RigidBodyOb {
 	
 	int flag;				/* (eRigidBodyOb_Flag) */
 	int col_groups;			/* Collision groups that determines wich rigid bodies can collide with each other */
-	int pad;
+	short mesh_source;		/* (eRigidBody_MeshSource) mesh source for mesh based collision shapes */
+	short pad;
 	
 	/* Physics Parameters */
 	float mass;				/* how much object 'weighs' (i.e. absolute 'amount of stuff' it holds) */
@@ -173,6 +174,15 @@ typedef enum eRigidBody_Shape {
 	//RB_SHAPE_COMPOUND,
 } eRigidBody_Shape;
 
+typedef enum eRigidBody_MeshSource {
+	/* base mesh */
+	RBO_MESH_BASE = 0,
+	/* only deformations */
+	RBO_MESH_DEFORM,
+	/* final derived mesh */
+	RBO_MESH_FINAL
+} eRigidBody_MeshSource;
+
 /* ******************************** */
 /* RigidBody Constraint */
 
diff --git a/source/blender/makesrna/intern/rna_rigidbody.c b/source/blender/makesrna/intern/rna_rigidbody.c
index cc14e60f45a..34b0f6a335a 100644
--- a/source/blender/makesrna/intern/rna_rigidbody.c
+++ b/source/blender/makesrna/intern/rna_rigidbody.c
@@ -76,6 +76,13 @@ EnumPropertyItem rigidbody_constraint_type_items[] = {
 	{RBC_TYPE_MOTOR, "MOTOR", ICON_NONE, "Motor", "Drive rigid body around or along an axis"},
 	{0, NULL, 0, NULL, NULL}};
 
+/* mesh source for collision shape creation */
+EnumPropertyItem rigidbody_mesh_source_items[] = {
+	{RBO_MESH_BASE, "BASE", 0, "Base", "Base mesh"},
+	{RBO_MESH_DEFORM, "DEFORM", 0, "Deform", "Deformations (shaps keys, deform modifiers"},
+	{RBO_MESH_FINAL, "FINAL", 0, "Final", "All modifiers"},
+	{0, NULL, 0, NULL, NULL}};
+
 
 #ifdef RNA_RUNTIME
 
@@ -769,6 +776,13 @@ static void rna_def_rigidbody_object(BlenderRNA *brna)
 	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
 	RNA_def_property_update(prop, NC_OBJECT | ND_POINTCACHE, "rna_RigidBodyOb_reset");
 	
+	prop = RNA_def_property(srna, "mesh_source", PROP_ENUM, PROP_NONE);
+	RNA_def_property_enum_sdna(prop, NULL, "mesh_source");
+	RNA_def_property_enum_items(prop, rigidbody_mesh_source_items);
+	RNA_def_property_ui_text(prop, "Mesh Source", "Source of the mesh used to create collision shape");
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+	RNA_def_property_update(prop, NC_OBJECT | ND_POINTCACHE, "rna_RigidBodyOb_reset");
+	
 	/* booleans */
 	prop = RNA_def_property(srna, "enabled", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_negative_sdna(prop, NULL, "flag", RBO_FLAG_DISABLED);