Collision: allow disabling collision without removing the modifier.

The `object.collision.use` flag was treated as a redundant marker
of the existence of the modifier, going as far as adding/removing
it when the value was changed, which is not actually very useful.
Removing the modifier loses its position in the stack, and requires
a dependency graph rebuild. It feels it may be a legacy flag?

What would be useful however is the ability to toggle collisions
dynamically without removing the modifier. This patch adjusts the
code to keep the modifier when the flag is disabled, and add it
if it doesn't exist when the flag is enabled. The modifier now
checks the flag at the start and quickly exits after cleaning
up stale data. The redesigned setting is exposed in the UI.

Collisions can't be disabled by simply using the modifier enable
flags because the modifier merely saves a snapshot of the mesh at
a certain point of the modifier stack for other objects to use,
and thus has to be able to clear the stale data.

Differential Revision: https://developer.blender.org/D10064

5 files changed, 23 insertions, 18 deletions

diff --git a/release/scripts/startup/bl_ui/properties_physics_common.py b/release/scripts/startup/bl_ui/properties_physics_common.py
index 1ffaa47168f..3ee6894f349 100644
--- a/release/scripts/startup/bl_ui/properties_physics_common.py
+++ b/release/scripts/startup/bl_ui/properties_physics_common.py
@@ -48,6 +48,7 @@ def physics_add(layout, md, name, type, typeicon, toggles):
         if toggles:
             row.prop(md, "show_viewport", text="")
             row.prop(md, "show_render", text="")
+        return row
     else:
         row.operator(
             "object.modifier_add",
@@ -89,7 +90,10 @@ class PHYSICS_PT_add(PhysicButtonsPanel, Panel):
             col.operator("object.forcefield_toggle", text="Force Field", icon='X')
 
         if obj.type == 'MESH':
-            physics_add(col, context.collision, "Collision", 'COLLISION', 'MOD_PHYSICS', False)
+            row = physics_add(col, context.collision, "Collision", 'COLLISION', 'MOD_PHYSICS', False)
+            if row and obj.collision:
+                row.prop(obj.collision, "use", text="", icon="HIDE_OFF" if obj.collision.use else "HIDE_ON")
+
             physics_add(col, context.cloth, "Cloth", 'CLOTH', 'MOD_CLOTH', True)
             physics_add(col, context.dynamic_paint, "Dynamic Paint", 'DYNAMIC_PAINT', 'MOD_DYNAMICPAINT', True)
 
diff --git a/source/blender/blenkernel/intern/collision.c b/source/blender/blenkernel/intern/collision.c
index 068216e3159..b4d668c7f50 100644
--- a/source/blender/blenkernel/intern/collision.c
+++ b/source/blender/blenkernel/intern/collision.c
@@ -1238,13 +1238,8 @@ static void add_collision_object(ListBase *relations,
                                  int level,
                                  unsigned int modifier_type)
 {
-  CollisionModifierData *cmd = NULL;
-
   /* only get objects with collision modifier */
-  if (((modifier_type == eModifierType_Collision) && ob->pd && ob->pd->deflect) ||
-      (modifier_type != eModifierType_Collision)) {
-    cmd = (CollisionModifierData *)BKE_modifiers_findby_type(ob, modifier_type);
-  }
+  ModifierData *cmd = BKE_modifiers_findby_type(ob, modifier_type);
 
   if (cmd) {
     CollisionRelation *relation = MEM_callocN(sizeof(CollisionRelation), "CollisionRelation");
@@ -1320,6 +1315,10 @@ Object **BKE_collision_objects_create(Depsgraph *depsgraph,
     /* Get evaluated object. */
     Object *ob = (Object *)DEG_get_evaluated_id(depsgraph, &relation->ob->id);
 
+    if (modifier_type == eModifierType_Collision && !(ob->pd && ob->pd->deflect)) {
+      continue;
+    }
+
     if (ob != self) {
       objects[num] = ob;
       num++;
diff --git a/source/blender/makesrna/intern/rna_object_force.c b/source/blender/makesrna/intern/rna_object_force.c
index 12fd2b78d91..14dd8a68fee 100644
--- a/source/blender/makesrna/intern/rna_object_force.c
+++ b/source/blender/makesrna/intern/rna_object_force.c
@@ -859,14 +859,12 @@ static void rna_CollisionSettings_dependency_update(Main *bmain, Scene *scene, P
   Object *ob = (Object *)ptr->owner_id;
   ModifierData *md = BKE_modifiers_findby_type(ob, eModifierType_Collision);
 
-  /* add/remove modifier as needed */
+  /* add the modifier if needed */
   if (ob->pd->deflect && !md) {
     ED_object_modifier_add(NULL, bmain, scene, ob, NULL, eModifierType_Collision);
   }
-  else if (!ob->pd->deflect && md) {
-    ED_object_modifier_remove(NULL, bmain, scene, ob, md);
-  }
 
+  DEG_id_tag_update(&ob->id, ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY);
   WM_main_add_notifier(NC_OBJECT | ND_DRAW, ob);
 }
 
diff --git a/source/blender/modifiers/intern/MOD_collision.c b/source/blender/modifiers/intern/MOD_collision.c
index a80bac7c6de..04b68a7a800 100644
--- a/source/blender/modifiers/intern/MOD_collision.c
+++ b/source/blender/modifiers/intern/MOD_collision.c
@@ -111,6 +111,16 @@ static void deformVerts(ModifierData *md,
   MVert *tempVert = NULL;
   Object *ob = ctx->object;
 
+  /* If collision is disabled, free the stale data and exit. */
+  if (!ob->pd || !ob->pd->deflect) {
+    if (!ob->pd) {
+      printf("CollisionModifier: collision settings are missing!\n");
+    }
+
+    freeData(md);
+    return;
+  }
+
   if (mesh == NULL) {
     mesh_src = MOD_deform_mesh_eval_get(ob, NULL, NULL, NULL, numVerts, false, false);
   }
@@ -120,11 +130,6 @@ static void deformVerts(ModifierData *md,
     mesh_src = (Mesh *)BKE_id_copy_ex(NULL, (ID *)mesh, NULL, LIB_ID_COPY_LOCALIZE);
   }
 
-  if (!ob->pd) {
-    printf("CollisionModifier deformVerts: Should not happen!\n");
-    return;
-  }
-
   if (mesh_src) {
     float current_time = 0;
     uint mvert_num = 0;
diff --git a/source/blender/modifiers/intern/MOD_ui_common.c b/source/blender/modifiers/intern/MOD_ui_common.c
index 58eeba35ca9..c5ee96491b8 100644
--- a/source/blender/modifiers/intern/MOD_ui_common.c
+++ b/source/blender/modifiers/intern/MOD_ui_common.c
@@ -351,8 +351,7 @@ static void modifier_panel_header(const bContext *C, Panel *panel)
     }
   }
   /* Collision and Surface are always enabled, hide buttons. */
-  if (((md->type != eModifierType_Collision) || !(ob->pd && ob->pd->deflect)) &&
-      (md->type != eModifierType_Surface)) {
+  if ((md->type != eModifierType_Collision) && (md->type != eModifierType_Surface)) {
     if (mti->flags & eModifierTypeFlag_SupportsEditmode) {
       sub = uiLayoutRow(row, true);
       uiLayoutSetActive(sub, (md->mode & eModifierMode_Realtime));