Depsgraph: support depending on collection geometry

This fixes T87666 and T83252.

The boolean modifier and geometry nodes can depend on the geometry
of an entire collection. Before, the modifiers had to manually create relations
to all the objects in the collection. This worked for the most part, but was
cumbersome and did not solve all issues. For example, the modifiers were not
properly updated when objects were added/removed from the referenced collection.

This commit introduces the concept of "collection geometry" in the depsgraph.
The geometry of a collection depends on the transforms and geometry of all
the objects in it. The boolean modifier and geometry nodes can now just depend
on the collection geometry instead of creating all the dependencies themselves.

Differential Revision: https://developer.blender.org/D11053

9 files changed, 87 insertions, 40 deletions

diff --git a/source/blender/blenkernel/intern/collection.c b/source/blender/blenkernel/intern/collection.c
index b29dd007c51..3170c3aa65c 100644
--- a/source/blender/blenkernel/intern/collection.c
+++ b/source/blender/blenkernel/intern/collection.c
@@ -1150,6 +1150,8 @@ bool BKE_collection_object_add(Main *bmain, Collection *collection, Object *ob)
     BKE_main_collection_sync(bmain);
   }
 
+  DEG_id_tag_update(&collection->id, ID_RECALC_GEOMETRY);
+
   return true;
 }
 
@@ -1201,6 +1203,8 @@ bool BKE_collection_object_remove(Main *bmain,
     BKE_main_collection_sync(bmain);
   }
 
+  DEG_id_tag_update(&collection->id, ID_RECALC_GEOMETRY);
+
   return true;
 }
 
diff --git a/source/blender/depsgraph/DEG_depsgraph_build.h b/source/blender/depsgraph/DEG_depsgraph_build.h
index 4e618d8625d..5f9e78837a7 100644
--- a/source/blender/depsgraph/DEG_depsgraph_build.h
+++ b/source/blender/depsgraph/DEG_depsgraph_build.h
@@ -40,6 +40,7 @@ struct Object;
 struct Scene;
 struct Simulation;
 struct bNodeTree;
+struct Collection;
 
 #include "BLI_sys_types.h"
 
@@ -137,6 +138,12 @@ void DEG_add_object_relation(struct DepsNodeHandle *node_handle,
                              struct Object *object,
                              eDepsObjectComponentType component,
                              const char *description);
+void DEG_add_collection_geometry_relation(struct DepsNodeHandle *node_handle,
+                                          struct Collection *collection,
+                                          const char *description);
+void DEG_add_collection_geometry_customdata_mask(struct DepsNodeHandle *node_handle,
+                                                 struct Collection *collection,
+                                                 const struct CustomData_MeshMasks *masks);
 void DEG_add_simulation_relation(struct DepsNodeHandle *node_handle,
                                  struct Simulation *simulation,
                                  const char *description);
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
index d8dc66883a0..ec5037fb29c 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
@@ -554,6 +554,7 @@ void DepsgraphNodeBuilder::build_collection(LayerCollection *from_layer_collecti
     id_node->is_directly_visible = is_collection_visible;
 
     build_idproperties(collection->id.properties);
+    add_operation_node(&collection->id, NodeType::GEOMETRY, OperationCode::GEOMETRY_EVAL_DONE);
   }
   if (from_layer_collection != nullptr) {
     /* If we came from layer collection we don't go deeper, view layer
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
index 3cc2ec02165..b8cab43f676 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
@@ -635,11 +635,38 @@ void DepsgraphRelationBuilder::build_collection(LayerCollection *from_layer_coll
   ComponentKey duplicator_key(object != nullptr ? &object->id : nullptr, NodeType::DUPLI);
   if (!group_done) {
     build_idproperties(collection->id.properties);
+    OperationKey collection_geometry_key{
+        &collection->id, NodeType::GEOMETRY, OperationCode::GEOMETRY_EVAL_DONE};
     LISTBASE_FOREACH (CollectionObject *, cob, &collection->gobject) {
       build_object(cob->ob);
+
+      /* The geometry of a collection depends on the positions of the elements in it. */
+      OperationKey object_transform_key{
+          &cob->ob->id, NodeType::TRANSFORM, OperationCode::TRANSFORM_FINAL};
+      add_relation(object_transform_key, collection_geometry_key, "Collection Geometry");
+
+      /* Only create geometry relations to child objects, if they have a geometry component. */
+      OperationKey object_geometry_key{
+          &cob->ob->id, NodeType::GEOMETRY, OperationCode::GEOMETRY_EVAL};
+      if (find_node(object_geometry_key) != nullptr) {
+        add_relation(object_geometry_key, collection_geometry_key, "Collection Geometry");
+      }
+
+      /* An instance is part of the geometry of the collection. */
+      if (cob->ob->type == OB_EMPTY) {
+        Collection *collection_instance = object->instance_collection;
+        if (collection_instance != nullptr) {
+          OperationKey collection_instance_key{
+              &collection_instance->id, NodeType::GEOMETRY, OperationCode::GEOMETRY_EVAL_DONE};
+          add_relation(collection_instance_key, collection_geometry_key, "Collection Geometry");
+        }
+      }
     }
     LISTBASE_FOREACH (CollectionChild *, child, &collection->children) {
       build_collection(nullptr, nullptr, child->collection);
+      OperationKey child_collection_geometry_key{
+          &child->collection->id, NodeType::GEOMETRY, OperationCode::GEOMETRY_EVAL_DONE};
+      add_relation(child_collection_geometry_key, collection_geometry_key, "Collection Geometry");
     }
   }
   if (object != nullptr) {
diff --git a/source/blender/depsgraph/intern/depsgraph_build.cc b/source/blender/depsgraph/intern/depsgraph_build.cc
index 6717ba521f6..6c1e91d068b 100644
--- a/source/blender/depsgraph/intern/depsgraph_build.cc
+++ b/source/blender/depsgraph/intern/depsgraph_build.cc
@@ -32,11 +32,13 @@
 #include "PIL_time_utildefines.h"
 
 #include "DNA_cachefile_types.h"
+#include "DNA_collection_types.h"
 #include "DNA_node_types.h"
 #include "DNA_object_types.h"
 #include "DNA_scene_types.h"
 #include "DNA_simulation_types.h"
 
+#include "BKE_collection.h"
 #include "BKE_main.h"
 #include "BKE_scene.h"
 
@@ -107,6 +109,29 @@ void DEG_add_object_relation(DepsNodeHandle *node_handle,
   deg_node_handle->builder->add_node_handle_relation(comp_key, deg_node_handle, description);
 }
 
+void DEG_add_collection_geometry_relation(DepsNodeHandle *node_handle,
+                                          Collection *collection,
+                                          const char *description)
+{
+  deg::OperationKey operation_key{
+      &collection->id, deg::NodeType::GEOMETRY, deg::OperationCode::GEOMETRY_EVAL_DONE};
+  deg::DepsNodeHandle *deg_node_handle = get_node_handle(node_handle);
+  deg_node_handle->builder->add_node_handle_relation(operation_key, deg_node_handle, description);
+}
+
+void DEG_add_collection_geometry_customdata_mask(DepsNodeHandle *node_handle,
+                                                 Collection *collection,
+                                                 const CustomData_MeshMasks *masks)
+{
+  FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN (collection, ob) {
+    DEG_add_customdata_mask(node_handle, ob, masks);
+    if (ob->type == OB_EMPTY && ob->instance_collection != nullptr) {
+      DEG_add_collection_geometry_customdata_mask(node_handle, ob->instance_collection, masks);
+    }
+  }
+  FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
+}
+
 void DEG_add_simulation_relation(DepsNodeHandle *node_handle,
                                  Simulation *simulation,
                                  const char *description)
diff --git a/source/blender/depsgraph/intern/depsgraph_tag.cc b/source/blender/depsgraph/intern/depsgraph_tag.cc
index b0a8e5d36e6..204143d7cbd 100644
--- a/source/blender/depsgraph/intern/depsgraph_tag.cc
+++ b/source/blender/depsgraph/intern/depsgraph_tag.cc
@@ -594,6 +594,7 @@ NodeType geometry_tag_to_component(const ID *id)
     case ID_HA:
     case ID_PT:
     case ID_VO:
+    case ID_GR:
       return NodeType::GEOMETRY;
     case ID_PA: /* Particles */
       return NodeType::UNDEFINED;
diff --git a/source/blender/makesdna/DNA_ID.h b/source/blender/makesdna/DNA_ID.h
index f9524fac72d..fa60bba54d5 100644
--- a/source/blender/makesdna/DNA_ID.h
+++ b/source/blender/makesdna/DNA_ID.h
@@ -591,14 +591,16 @@ typedef enum IDRecalcFlag {
   /* ** Object transformation changed. ** */
   ID_RECALC_TRANSFORM = (1 << 0),
 
-  /* ** Object geometry changed. **
+  /* ** Geometry changed. **
    *
    * When object of armature type gets tagged with this flag, its pose is
    * re-evaluated.
    * When object of other type is tagged with this flag it makes the modifier
    * stack to be re-evaluated.
    * When object data type (mesh, curve, ...) gets tagged with this flag it
-   * makes all objects which shares this data-block to be updated. */
+   * makes all objects which shares this data-block to be updated.
+   * When a collection gets tagged with this flag, all objects depending on the geometry and
+   * transforms on any of the objects in the collection are updated. */
   ID_RECALC_GEOMETRY = (1 << 1),
 
   /* ** Animation or time changed and animation is to be re-evaluated. ** */
diff --git a/source/blender/modifiers/intern/MOD_boolean.cc b/source/blender/modifiers/intern/MOD_boolean.cc
index c49bcce2dd3..40d5386c2fa 100644
--- a/source/blender/modifiers/intern/MOD_boolean.cc
+++ b/source/blender/modifiers/intern/MOD_boolean.cc
@@ -124,13 +124,7 @@ static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphConte
   Collection *col = bmd->collection;
 
   if ((bmd->flag & eBooleanModifierFlag_Collection) && col != nullptr) {
-    FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN (col, operand_ob) {
-      if (operand_ob->type == OB_MESH && operand_ob != ctx->object) {
-        DEG_add_object_relation(ctx->node, operand_ob, DEG_OB_COMP_TRANSFORM, "Boolean Modifier");
-        DEG_add_object_relation(ctx->node, operand_ob, DEG_OB_COMP_GEOMETRY, "Boolean Modifier");
-      }
-    }
-    FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
+    DEG_add_collection_geometry_relation(ctx->node, col, "Boolean Modifier");
   }
   /* We need own transformation as well. */
   DEG_add_modifier_to_transform_relation(ctx->node, "Boolean Modifier");
diff --git a/source/blender/modifiers/intern/MOD_nodes.cc b/source/blender/modifiers/intern/MOD_nodes.cc
index 57de629fc18..607b05d39b0 100644
--- a/source/blender/modifiers/intern/MOD_nodes.cc
+++ b/source/blender/modifiers/intern/MOD_nodes.cc
@@ -164,48 +164,34 @@ static void find_used_ids_from_settings(const NodesModifierSettings &settings, S
       &ids);
 }
 
-static void add_collection_object_relations_recursive(const ModifierUpdateDepsgraphContext *ctx,
-                                                      Collection &collection);
+/* We don't know exactly what attributes from the other object we will need. */
+static const CustomData_MeshMasks dependency_data_mask{CD_MASK_PROP_ALL | CD_MASK_MDEFORMVERT,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL};
+
+static void add_collection_relation(const ModifierUpdateDepsgraphContext *ctx,
+                                    Collection &collection)
+{
+  DEG_add_collection_geometry_relation(ctx->node, &collection, "Nodes Modifier");
+  DEG_add_collection_geometry_customdata_mask(ctx->node, &collection, &dependency_data_mask);
+}
 
 static void add_object_relation(const ModifierUpdateDepsgraphContext *ctx, Object &object)
 {
   DEG_add_object_relation(ctx->node, &object, DEG_OB_COMP_TRANSFORM, "Nodes Modifier");
   if (&(ID &)object != &ctx->object->id) {
-    if (object.type == OB_EMPTY) {
-      Collection *collection_instance = object.instance_collection;
-      if (collection_instance != nullptr) {
-        add_collection_object_relations_recursive(ctx, *collection_instance);
-      }
+    if (object.type == OB_EMPTY && object.instance_collection != nullptr) {
+      add_collection_relation(ctx, *object.instance_collection);
     }
     else if (ELEM(object.type, OB_MESH, OB_POINTCLOUD, OB_VOLUME)) {
       DEG_add_object_relation(ctx->node, &object, DEG_OB_COMP_GEOMETRY, "Nodes Modifier");
-      /* We don't know exactly what attributes from the other object we will need. */
-      CustomData_MeshMasks mask;
-      mask.vmask = CD_MASK_PROP_ALL | CD_MASK_MDEFORMVERT;
-      mask.pmask = CD_MASK_PROP_ALL;
-      mask.lmask = CD_MASK_PROP_ALL;
-      mask.fmask = CD_MASK_PROP_ALL;
-      mask.emask = CD_MASK_PROP_ALL;
-      DEG_add_customdata_mask(ctx->node, &object, &mask);
+      DEG_add_customdata_mask(ctx->node, &object, &dependency_data_mask);
     }
   }
 }
 
-static void add_collection_object_relations_recursive(const ModifierUpdateDepsgraphContext *ctx,
-                                                      Collection &collection)
-{
-  LISTBASE_FOREACH (CollectionObject *, collection_object, &collection.gobject) {
-    BLI_assert(collection_object->ob != nullptr);
-    Object &object = *collection_object->ob;
-    add_object_relation(ctx, object);
-  }
-  LISTBASE_FOREACH (CollectionChild *, collection_child, &collection.children) {
-    BLI_assert(collection_child->collection != nullptr);
-    Collection &collection = *collection_child->collection;
-    add_collection_object_relations_recursive(ctx, collection);
-  }
-}
-
 static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphContext *ctx)
 {
   NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
@@ -223,7 +209,7 @@ static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphConte
       }
       if (GS(id->name) == ID_GR) {
         Collection *collection = reinterpret_cast<Collection *>(id);
-        add_collection_object_relations_recursive(ctx, *collection);
+        add_collection_relation(ctx, *collection);
       }
     }
   }