Depsgraph: support depending on collection geometry

This fixes T87666 and T83252.

The boolean modifier and geometry nodes can depend on the geometry
of an entire collection. Before, the modifiers had to manually create relations
to all the objects in the collection. This worked for the most part, but was
cumbersome and did not solve all issues. For example, the modifiers were not
properly updated when objects were added/removed from the referenced collection.

This commit introduces the concept of "collection geometry" in the depsgraph.
The geometry of a collection depends on the transforms and geometry of all
the objects in it. The boolean modifier and geometry nodes can now just depend
on the collection geometry instead of creating all the dependencies themselves.

Differential Revision: https://developer.blender.org/D11053

2 files changed, 18 insertions, 38 deletions

diff --git a/source/blender/modifiers/intern/MOD_boolean.cc b/source/blender/modifiers/intern/MOD_boolean.cc
index c49bcce2dd3..40d5386c2fa 100644
--- a/source/blender/modifiers/intern/MOD_boolean.cc
+++ b/source/blender/modifiers/intern/MOD_boolean.cc
@@ -124,13 +124,7 @@ static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphConte
   Collection *col = bmd->collection;
 
   if ((bmd->flag & eBooleanModifierFlag_Collection) && col != nullptr) {
-    FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN (col, operand_ob) {
-      if (operand_ob->type == OB_MESH && operand_ob != ctx->object) {
-        DEG_add_object_relation(ctx->node, operand_ob, DEG_OB_COMP_TRANSFORM, "Boolean Modifier");
-        DEG_add_object_relation(ctx->node, operand_ob, DEG_OB_COMP_GEOMETRY, "Boolean Modifier");
-      }
-    }
-    FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
+    DEG_add_collection_geometry_relation(ctx->node, col, "Boolean Modifier");
   }
   /* We need own transformation as well. */
   DEG_add_modifier_to_transform_relation(ctx->node, "Boolean Modifier");
diff --git a/source/blender/modifiers/intern/MOD_nodes.cc b/source/blender/modifiers/intern/MOD_nodes.cc
index 57de629fc18..607b05d39b0 100644
--- a/source/blender/modifiers/intern/MOD_nodes.cc
+++ b/source/blender/modifiers/intern/MOD_nodes.cc
@@ -164,48 +164,34 @@ static void find_used_ids_from_settings(const NodesModifierSettings &settings, S
       &ids);
 }
 
-static void add_collection_object_relations_recursive(const ModifierUpdateDepsgraphContext *ctx,
-                                                      Collection &collection);
+/* We don't know exactly what attributes from the other object we will need. */
+static const CustomData_MeshMasks dependency_data_mask{CD_MASK_PROP_ALL | CD_MASK_MDEFORMVERT,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL,
+                                                       CD_MASK_PROP_ALL};
+
+static void add_collection_relation(const ModifierUpdateDepsgraphContext *ctx,
+                                    Collection &collection)
+{
+  DEG_add_collection_geometry_relation(ctx->node, &collection, "Nodes Modifier");
+  DEG_add_collection_geometry_customdata_mask(ctx->node, &collection, &dependency_data_mask);
+}
 
 static void add_object_relation(const ModifierUpdateDepsgraphContext *ctx, Object &object)
 {
   DEG_add_object_relation(ctx->node, &object, DEG_OB_COMP_TRANSFORM, "Nodes Modifier");
   if (&(ID &)object != &ctx->object->id) {
-    if (object.type == OB_EMPTY) {
-      Collection *collection_instance = object.instance_collection;
-      if (collection_instance != nullptr) {
-        add_collection_object_relations_recursive(ctx, *collection_instance);
-      }
+    if (object.type == OB_EMPTY && object.instance_collection != nullptr) {
+      add_collection_relation(ctx, *object.instance_collection);
     }
     else if (ELEM(object.type, OB_MESH, OB_POINTCLOUD, OB_VOLUME)) {
       DEG_add_object_relation(ctx->node, &object, DEG_OB_COMP_GEOMETRY, "Nodes Modifier");
-      /* We don't know exactly what attributes from the other object we will need. */
-      CustomData_MeshMasks mask;
-      mask.vmask = CD_MASK_PROP_ALL | CD_MASK_MDEFORMVERT;
-      mask.pmask = CD_MASK_PROP_ALL;
-      mask.lmask = CD_MASK_PROP_ALL;
-      mask.fmask = CD_MASK_PROP_ALL;
-      mask.emask = CD_MASK_PROP_ALL;
-      DEG_add_customdata_mask(ctx->node, &object, &mask);
+      DEG_add_customdata_mask(ctx->node, &object, &dependency_data_mask);
     }
   }
 }
 
-static void add_collection_object_relations_recursive(const ModifierUpdateDepsgraphContext *ctx,
-                                                      Collection &collection)
-{
-  LISTBASE_FOREACH (CollectionObject *, collection_object, &collection.gobject) {
-    BLI_assert(collection_object->ob != nullptr);
-    Object &object = *collection_object->ob;
-    add_object_relation(ctx, object);
-  }
-  LISTBASE_FOREACH (CollectionChild *, collection_child, &collection.children) {
-    BLI_assert(collection_child->collection != nullptr);
-    Collection &collection = *collection_child->collection;
-    add_collection_object_relations_recursive(ctx, collection);
-  }
-}
-
 static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphContext *ctx)
 {
   NodesModifierData *nmd = reinterpret_cast<NodesModifierData *>(md);
@@ -223,7 +209,7 @@ static void updateDepsgraph(ModifierData *md, const ModifierUpdateDepsgraphConte
       }
       if (GS(id->name) == ID_GR) {
         Collection *collection = reinterpret_cast<Collection *>(id);
-        add_collection_object_relations_recursive(ctx, *collection);
+        add_collection_relation(ctx, *collection);
       }
     }
   }