Depsgraph: Save memory by ignoring invisible objects

This finished old standing TODO which was attempting to
ignore objects of all invisible collections.

The difference here is that we remove invisible bases from
view layers. This guarantees that the evaluated state is
consistent and does not reference original objects.

6 files changed, 60 insertions, 25 deletions

diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
index 637fd5887a0..c705f229669 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
@@ -419,7 +419,7 @@ void DepsgraphNodeBuilder::build_id(ID *id)
 			build_camera((Camera *)id);
 			break;
 		case ID_GR:
-			build_collection((Collection *)id);
+			build_collection(NULL, (Collection *)id);
 			break;
 		case ID_OB:
 			/* TODO(sergey): Get visibility from a "parent" somehow.
@@ -489,7 +489,9 @@ void DepsgraphNodeBuilder::build_id(ID *id)
 	}
 }
 
-void DepsgraphNodeBuilder::build_collection(Collection *collection)
+void DepsgraphNodeBuilder::build_collection(
+        LayerCollection *from_layer_collection,
+        Collection *collection)
 {
 	const int restrict_flag = (graph_->mode == DAG_EVAL_VIEWPORT)
 	        ? COLLECTION_RESTRICT_VIEW
@@ -497,13 +499,16 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	const bool is_collection_restricted = (collection->flag & restrict_flag);
 	const bool is_collection_visible =
 	        !is_collection_restricted && is_parent_collection_visible_;
+	IDDepsNode *id_node;
 	if (built_map_.checkIsBuiltAndTag(collection)) {
-		IDDepsNode *id_node = find_id_node(&collection->id);
-		if (is_collection_visible && !id_node->is_directly_visible) {
+		id_node = find_id_node(&collection->id);
+		if (is_collection_visible &&
+		    id_node->is_directly_visible == false &&
+			id_node->is_collection_fully_expanded == true)
+		{
 			/* Collection became visible, make sure nested collections and
 			 * objects are poked with the new visibility flag, since they
-			 * might become visible too.
-			 */
+			 * might become visible too. */
 		}
 		else {
 			return;
@@ -511,9 +516,14 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	}
 	else {
 		/* Collection itself. */
-		IDDepsNode *id_node = add_id_node(&collection->id);
+		id_node = add_id_node(&collection->id);
 		id_node->is_directly_visible = is_collection_visible;
 	}
+	if (from_layer_collection != NULL) {
+		/* If we came from layer collection we don't go deeper, view layer
+		 * builder takes care of going deeper. */
+		return;
+	}
 	/* Backup state. */
 	Collection *current_state_collection = collection_;
 	const bool is_current_parent_collection_visible =
@@ -528,11 +538,12 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	}
 	/* Build child collections. */
 	LISTBASE_FOREACH (CollectionChild *, child, &collection->children) {
-		build_collection(child->collection);
+		build_collection(NULL, child->collection);
 	}
 	/* Restore state. */
 	collection_ = current_state_collection;
 	is_parent_collection_visible_ = is_current_parent_collection_visible;
+	id_node->is_collection_fully_expanded = true;
 }
 
 void DepsgraphNodeBuilder::build_object(int base_index,
@@ -635,7 +646,7 @@ void DepsgraphNodeBuilder::build_object(int base_index,
 		const bool is_current_parent_collection_visible =
 		        is_parent_collection_visible_;
 		is_parent_collection_visible_ = is_visible;
-		build_collection(object->dup_group);
+		build_collection(NULL, object->dup_group);
 		is_parent_collection_visible_ = is_current_parent_collection_visible;
 		add_operation_node(&object->id,
 		                   DEG_NODE_TYPE_DUPLI,
@@ -1059,7 +1070,7 @@ void DepsgraphNodeBuilder::build_rigidbody(Scene *scene)
 
 	/* objects - simulation participants */
 	if (rbw->group) {
-		build_collection(rbw->group);
+		build_collection(NULL, rbw->group);
 
 		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(rbw->group, object)
 		{
@@ -1137,7 +1148,7 @@ void DepsgraphNodeBuilder::build_particles(Object *object,
 				break;
 			case PART_DRAW_GR:
 				if (part->dup_group != NULL) {
-					build_collection(part->dup_group);
+					build_collection(NULL, part->dup_group);
 				}
 				break;
 		}
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.h b/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
index 3c0c5f749ca..3b795bf9d58 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
@@ -165,7 +165,8 @@ struct DepsgraphNodeBuilder {
 	void build_view_layer(Scene *scene,
 	                      ViewLayer *view_layer,
 	                      eDepsNode_LinkedState_Type linked_state);
-	void build_collection(Collection *collection);
+	void build_collection(LayerCollection *from_layer_collection,
+	                      Collection *collection);
 	void build_object(int base_index,
 	                  Object *object,
 	                  eDepsNode_LinkedState_Type linked_state,
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
index 5efa427fcdb..70bd533647c 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
@@ -75,7 +75,7 @@ void DepsgraphNodeBuilder::build_layer_collections(ListBase *lb)
 			continue;
 		}
 		if ((lc->flag & LAYER_COLLECTION_EXCLUDE) == 0) {
-			build_collection(lc->collection);
+			build_collection(lc, lc->collection);
 		}
 		build_layer_collections(&lc->layer_collections);
 	}
@@ -86,8 +86,9 @@ void DepsgraphNodeBuilder::build_view_layer(
         ViewLayer *view_layer,
         eDepsNode_LinkedState_Type linked_state)
 {
-	view_layer_index_ = BLI_findindex(&scene->view_layers, view_layer);
-	BLI_assert(view_layer_index_ != -1);
+	/* NOTE: Pass view layer index of 0 since after scene CoW there is
+	 * only one view layer in there. */
+	view_layer_index_ = 0;
 	/* Scene ID block. */
 	add_id_node(&scene->id);
 	/* Time source. */
@@ -109,9 +110,14 @@ void DepsgraphNodeBuilder::build_view_layer(
 	LISTBASE_FOREACH(Base *, base, &view_layer->object_bases) {
 		/* object itself */
 		const bool is_object_visible = (base->flag & base_flag);
-		build_object(base_index, base->object, linked_state, is_object_visible);
+		if (is_object_visible) {
+			build_object(base_index,
+			             base->object,
+			             linked_state,
+			             is_object_visible);
+			++base_index;
+		}
 		base->object->select_color = select_color++;
-		++base_index;
 	}
 	build_layer_collections(&view_layer->layer_collections);
 	if (scene->camera != NULL) {
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
index 8080376488b..3fb6138d001 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
@@ -434,7 +434,7 @@ void DepsgraphRelationBuilder::build_id(ID *id)
 			build_camera((Camera *)id);
 			break;
 		case ID_GR:
-			build_collection(NULL, (Collection *)id);
+			build_collection(NULL, NULL, (Collection *)id);
 			break;
 		case ID_OB:
 			build_object(NULL, (Object *)id);
@@ -486,9 +486,19 @@ void DepsgraphRelationBuilder::build_id(ID *id)
 }
 
 void DepsgraphRelationBuilder::build_collection(
+        LayerCollection *from_layer_collection,
         Object *object,
         Collection *collection)
 {
+	if (from_layer_collection != NULL) {
+		/* If we came from layer collection we don't go deeper, view layer
+		 * builder takes care of going deeper.
+		 *
+		 * NOTE: Do early output before tagging build as done, so possbile
+		 * subsequent builds from outside of the layer collection properly
+		 * recurses into all the nested objects and collections. */
+		return;
+	}
 	const bool group_done = built_map_.checkIsBuiltAndTag(collection);
 	OperationKey object_transform_final_key(object != NULL ? &object->id : NULL,
 	                                        DEG_NODE_TYPE_TRANSFORM,
@@ -500,7 +510,7 @@ void DepsgraphRelationBuilder::build_collection(
 			build_object(NULL, cob->ob);
 		}
 		LISTBASE_FOREACH (CollectionChild *, child, &collection->children) {
-			build_collection(NULL, child->collection);
+			build_collection(NULL, NULL, child->collection);
 		}
 	}
 	if (object != NULL) {
@@ -644,7 +654,7 @@ void DepsgraphRelationBuilder::build_object(Base *base, Object *object)
 	}
 	/* Object dupligroup. */
 	if (object->dup_group != NULL) {
-		build_collection(object, object->dup_group);
+		build_collection(NULL, object, object->dup_group);
 	}
 	/* Point caches. */
 	build_object_pointcache(object);
@@ -1576,7 +1586,7 @@ void DepsgraphRelationBuilder::build_rigidbody(Scene *scene)
 
 	/* objects - simulation participants */
 	if (rbw->group) {
-		build_collection(NULL, rbw->group);
+		build_collection(NULL, NULL, rbw->group);
 
 		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(rbw->group, object)
 		{
@@ -1758,7 +1768,7 @@ void DepsgraphRelationBuilder::build_particles(Object *object)
 				break;
 			case PART_DRAW_GR:
 				if (part->dup_group != NULL) {
-					build_collection(NULL, part->dup_group);
+					build_collection(NULL, NULL, part->dup_group);
 					LISTBASE_FOREACH (CollectionObject *, go, &part->dup_group->gobject) {
 						build_particles_visualization_object(object,
 						                                     psys,
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.h b/source/blender/depsgraph/intern/builder/deg_builder_relations.h
index e86c6504693..81f21f5bf77 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.h
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.h
@@ -206,7 +206,9 @@ struct DepsgraphRelationBuilder
 	void build_id(ID *id);
 	void build_layer_collections(ListBase *lb);
 	void build_view_layer(Scene *scene, ViewLayer *view_layer);
-	void build_collection(Object *object, Collection *collection);
+	void build_collection(LayerCollection *from_layer_collection,
+	                      Object *object,
+	                      Collection *collection);
 	void build_object(Base *base, Object *object);
 	void build_object_flags(Base *base, Object *object);
 	void build_object_data(Object *object);
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
index 360e823d1bd..999508cffed 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
@@ -79,7 +79,7 @@ void DepsgraphRelationBuilder::build_layer_collections(ListBase *lb)
 			continue;
 		}
 		if ((lc->flag & LAYER_COLLECTION_EXCLUDE) == 0) {
-			build_collection(NULL, lc->collection);
+			build_collection(lc, NULL, lc->collection);
 		}
 		build_layer_collections(&lc->layer_collections);
 	}
@@ -94,8 +94,13 @@ void DepsgraphRelationBuilder::build_view_layer(Scene *scene, ViewLayer *view_la
 	 * passed to the evaluation functions. During relations builder we only
 	 * do NULL-pointer check of the base, so it's fine to pass original one.
 	 */
+	const int base_flag = (graph_->mode == DAG_EVAL_VIEWPORT) ?
+		BASE_ENABLED_VIEWPORT : BASE_ENABLED_RENDER;
 	LISTBASE_FOREACH (Base *, base, &view_layer->object_bases) {
-		build_object(base, base->object);
+		const bool is_object_visible = (base->flag & base_flag);
+		if (is_object_visible) {
+			build_object(base, base->object);
+		}
 	}
 
 	build_layer_collections(&view_layer->layer_collections);