Depsgraph: Save memory by ignoring invisible objects

This finished old standing TODO which was attempting to
ignore objects of all invisible collections.

The difference here is that we remove invisible bases from
view layers. This guarantees that the evaluated state is
consistent and does not reference original objects.

10 files changed, 143 insertions, 35 deletions

diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index fbfa47c8939..0e4a38503ae 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -1425,8 +1425,9 @@ void BKE_layer_eval_view_layer(
 
 	/* Visibility based on depsgraph mode. */
 	const eEvaluationMode mode = DEG_get_mode(depsgraph);
-	const int base_flag = (mode == DAG_EVAL_VIEWPORT) ? BASE_ENABLED_VIEWPORT : BASE_ENABLED_RENDER;
-
+	const int base_visible_flag = (mode == DAG_EVAL_VIEWPORT)
+	        ? BASE_ENABLED_VIEWPORT
+	        : BASE_ENABLED_RENDER;
 	/* Create array of bases, for fast index-based lookup. */
 	const int num_object_bases = BLI_listbase_count(&view_layer->object_bases);
 	MEM_SAFE_FREE(view_layer->object_bases_array);
@@ -1435,9 +1436,8 @@ void BKE_layer_eval_view_layer(
 	int base_index = 0;
 	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
 		/* Compute visibility for depsgraph evaluation mode. */
-		if (base->flag & base_flag) {
+		if (base->flag & base_visible_flag) {
 			base->flag |= BASE_ENABLED | BASE_VISIBLE;
-
 			if (mode == DAG_EVAL_VIEWPORT && (base->flag & BASE_HIDDEN)) {
 				base->flag &= ~BASE_VISIBLE;
 			}
@@ -1445,24 +1445,23 @@ void BKE_layer_eval_view_layer(
 		else {
 			base->flag &= ~(BASE_ENABLED | BASE_VISIBLE | BASE_SELECTABLE);
 		}
-
 		/* If base is not selectabled, clear select. */
 		if ((base->flag & BASE_SELECTABLE) == 0) {
 			base->flag &= ~BASE_SELECTED;
 		}
-
 		view_layer->object_bases_array[base_index++] = base;
 	}
-
 	/* Flush back base flag to the original view layer for editing. */
 	if (view_layer == DEG_get_evaluated_view_layer(depsgraph)) {
 		ViewLayer *view_layer_orig = DEG_get_input_view_layer(depsgraph);
 		Base *base_orig = view_layer_orig->object_bases.first;
 		const Base *base_eval = view_layer->object_bases.first;
 		while (base_orig != NULL) {
-			base_orig->flag = base_eval->flag;
+			if (base_orig->flag & base_visible_flag) {
+				base_orig->flag = base_eval->flag;
+				base_eval = base_eval->next;
+			}
 			base_orig = base_orig->next;
-			base_eval = base_eval->next;
 		}
 	}
 }
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
index 637fd5887a0..c705f229669 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
@@ -419,7 +419,7 @@ void DepsgraphNodeBuilder::build_id(ID *id)
 			build_camera((Camera *)id);
 			break;
 		case ID_GR:
-			build_collection((Collection *)id);
+			build_collection(NULL, (Collection *)id);
 			break;
 		case ID_OB:
 			/* TODO(sergey): Get visibility from a "parent" somehow.
@@ -489,7 +489,9 @@ void DepsgraphNodeBuilder::build_id(ID *id)
 	}
 }
 
-void DepsgraphNodeBuilder::build_collection(Collection *collection)
+void DepsgraphNodeBuilder::build_collection(
+        LayerCollection *from_layer_collection,
+        Collection *collection)
 {
 	const int restrict_flag = (graph_->mode == DAG_EVAL_VIEWPORT)
 	        ? COLLECTION_RESTRICT_VIEW
@@ -497,13 +499,16 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	const bool is_collection_restricted = (collection->flag & restrict_flag);
 	const bool is_collection_visible =
 	        !is_collection_restricted && is_parent_collection_visible_;
+	IDDepsNode *id_node;
 	if (built_map_.checkIsBuiltAndTag(collection)) {
-		IDDepsNode *id_node = find_id_node(&collection->id);
-		if (is_collection_visible && !id_node->is_directly_visible) {
+		id_node = find_id_node(&collection->id);
+		if (is_collection_visible &&
+		    id_node->is_directly_visible == false &&
+			id_node->is_collection_fully_expanded == true)
+		{
 			/* Collection became visible, make sure nested collections and
 			 * objects are poked with the new visibility flag, since they
-			 * might become visible too.
-			 */
+			 * might become visible too. */
 		}
 		else {
 			return;
@@ -511,9 +516,14 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	}
 	else {
 		/* Collection itself. */
-		IDDepsNode *id_node = add_id_node(&collection->id);
+		id_node = add_id_node(&collection->id);
 		id_node->is_directly_visible = is_collection_visible;
 	}
+	if (from_layer_collection != NULL) {
+		/* If we came from layer collection we don't go deeper, view layer
+		 * builder takes care of going deeper. */
+		return;
+	}
 	/* Backup state. */
 	Collection *current_state_collection = collection_;
 	const bool is_current_parent_collection_visible =
@@ -528,11 +538,12 @@ void DepsgraphNodeBuilder::build_collection(Collection *collection)
 	}
 	/* Build child collections. */
 	LISTBASE_FOREACH (CollectionChild *, child, &collection->children) {
-		build_collection(child->collection);
+		build_collection(NULL, child->collection);
 	}
 	/* Restore state. */
 	collection_ = current_state_collection;
 	is_parent_collection_visible_ = is_current_parent_collection_visible;
+	id_node->is_collection_fully_expanded = true;
 }
 
 void DepsgraphNodeBuilder::build_object(int base_index,
@@ -635,7 +646,7 @@ void DepsgraphNodeBuilder::build_object(int base_index,
 		const bool is_current_parent_collection_visible =
 		        is_parent_collection_visible_;
 		is_parent_collection_visible_ = is_visible;
-		build_collection(object->dup_group);
+		build_collection(NULL, object->dup_group);
 		is_parent_collection_visible_ = is_current_parent_collection_visible;
 		add_operation_node(&object->id,
 		                   DEG_NODE_TYPE_DUPLI,
@@ -1059,7 +1070,7 @@ void DepsgraphNodeBuilder::build_rigidbody(Scene *scene)
 
 	/* objects - simulation participants */
 	if (rbw->group) {
-		build_collection(rbw->group);
+		build_collection(NULL, rbw->group);
 
 		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(rbw->group, object)
 		{
@@ -1137,7 +1148,7 @@ void DepsgraphNodeBuilder::build_particles(Object *object,
 				break;
 			case PART_DRAW_GR:
 				if (part->dup_group != NULL) {
-					build_collection(part->dup_group);
+					build_collection(NULL, part->dup_group);
 				}
 				break;
 		}
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.h b/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
index 3c0c5f749ca..3b795bf9d58 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.h
@@ -165,7 +165,8 @@ struct DepsgraphNodeBuilder {
 	void build_view_layer(Scene *scene,
 	                      ViewLayer *view_layer,
 	                      eDepsNode_LinkedState_Type linked_state);
-	void build_collection(Collection *collection);
+	void build_collection(LayerCollection *from_layer_collection,
+	                      Collection *collection);
 	void build_object(int base_index,
 	                  Object *object,
 	                  eDepsNode_LinkedState_Type linked_state,
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
index 5efa427fcdb..70bd533647c 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
@@ -75,7 +75,7 @@ void DepsgraphNodeBuilder::build_layer_collections(ListBase *lb)
 			continue;
 		}
 		if ((lc->flag & LAYER_COLLECTION_EXCLUDE) == 0) {
-			build_collection(lc->collection);
+			build_collection(lc, lc->collection);
 		}
 		build_layer_collections(&lc->layer_collections);
 	}
@@ -86,8 +86,9 @@ void DepsgraphNodeBuilder::build_view_layer(
         ViewLayer *view_layer,
         eDepsNode_LinkedState_Type linked_state)
 {
-	view_layer_index_ = BLI_findindex(&scene->view_layers, view_layer);
-	BLI_assert(view_layer_index_ != -1);
+	/* NOTE: Pass view layer index of 0 since after scene CoW there is
+	 * only one view layer in there. */
+	view_layer_index_ = 0;
 	/* Scene ID block. */
 	add_id_node(&scene->id);
 	/* Time source. */
@@ -109,9 +110,14 @@ void DepsgraphNodeBuilder::build_view_layer(
 	LISTBASE_FOREACH(Base *, base, &view_layer->object_bases) {
 		/* object itself */
 		const bool is_object_visible = (base->flag & base_flag);
-		build_object(base_index, base->object, linked_state, is_object_visible);
+		if (is_object_visible) {
+			build_object(base_index,
+			             base->object,
+			             linked_state,
+			             is_object_visible);
+			++base_index;
+		}
 		base->object->select_color = select_color++;
-		++base_index;
 	}
 	build_layer_collections(&view_layer->layer_collections);
 	if (scene->camera != NULL) {
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
index 8080376488b..3fb6138d001 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
@@ -434,7 +434,7 @@ void DepsgraphRelationBuilder::build_id(ID *id)
 			build_camera((Camera *)id);
 			break;
 		case ID_GR:
-			build_collection(NULL, (Collection *)id);
+			build_collection(NULL, NULL, (Collection *)id);
 			break;
 		case ID_OB:
 			build_object(NULL, (Object *)id);
@@ -486,9 +486,19 @@ void DepsgraphRelationBuilder::build_id(ID *id)
 }
 
 void DepsgraphRelationBuilder::build_collection(
+        LayerCollection *from_layer_collection,
         Object *object,
         Collection *collection)
 {
+	if (from_layer_collection != NULL) {
+		/* If we came from layer collection we don't go deeper, view layer
+		 * builder takes care of going deeper.
+		 *
+		 * NOTE: Do early output before tagging build as done, so possbile
+		 * subsequent builds from outside of the layer collection properly
+		 * recurses into all the nested objects and collections. */
+		return;
+	}
 	const bool group_done = built_map_.checkIsBuiltAndTag(collection);
 	OperationKey object_transform_final_key(object != NULL ? &object->id : NULL,
 	                                        DEG_NODE_TYPE_TRANSFORM,
@@ -500,7 +510,7 @@ void DepsgraphRelationBuilder::build_collection(
 			build_object(NULL, cob->ob);
 		}
 		LISTBASE_FOREACH (CollectionChild *, child, &collection->children) {
-			build_collection(NULL, child->collection);
+			build_collection(NULL, NULL, child->collection);
 		}
 	}
 	if (object != NULL) {
@@ -644,7 +654,7 @@ void DepsgraphRelationBuilder::build_object(Base *base, Object *object)
 	}
 	/* Object dupligroup. */
 	if (object->dup_group != NULL) {
-		build_collection(object, object->dup_group);
+		build_collection(NULL, object, object->dup_group);
 	}
 	/* Point caches. */
 	build_object_pointcache(object);
@@ -1576,7 +1586,7 @@ void DepsgraphRelationBuilder::build_rigidbody(Scene *scene)
 
 	/* objects - simulation participants */
 	if (rbw->group) {
-		build_collection(NULL, rbw->group);
+		build_collection(NULL, NULL, rbw->group);
 
 		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(rbw->group, object)
 		{
@@ -1758,7 +1768,7 @@ void DepsgraphRelationBuilder::build_particles(Object *object)
 				break;
 			case PART_DRAW_GR:
 				if (part->dup_group != NULL) {
-					build_collection(NULL, part->dup_group);
+					build_collection(NULL, NULL, part->dup_group);
 					LISTBASE_FOREACH (CollectionObject *, go, &part->dup_group->gobject) {
 						build_particles_visualization_object(object,
 						                                     psys,
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.h b/source/blender/depsgraph/intern/builder/deg_builder_relations.h
index e86c6504693..81f21f5bf77 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.h
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.h
@@ -206,7 +206,9 @@ struct DepsgraphRelationBuilder
 	void build_id(ID *id);
 	void build_layer_collections(ListBase *lb);
 	void build_view_layer(Scene *scene, ViewLayer *view_layer);
-	void build_collection(Object *object, Collection *collection);
+	void build_collection(LayerCollection *from_layer_collection,
+	                      Object *object,
+	                      Collection *collection);
 	void build_object(Base *base, Object *object);
 	void build_object_flags(Base *base, Object *object);
 	void build_object_data(Object *object);
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
index 360e823d1bd..999508cffed 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
@@ -79,7 +79,7 @@ void DepsgraphRelationBuilder::build_layer_collections(ListBase *lb)
 			continue;
 		}
 		if ((lc->flag & LAYER_COLLECTION_EXCLUDE) == 0) {
-			build_collection(NULL, lc->collection);
+			build_collection(lc, NULL, lc->collection);
 		}
 		build_layer_collections(&lc->layer_collections);
 	}
@@ -94,8 +94,13 @@ void DepsgraphRelationBuilder::build_view_layer(Scene *scene, ViewLayer *view_la
 	 * passed to the evaluation functions. During relations builder we only
 	 * do NULL-pointer check of the base, so it's fine to pass original one.
 	 */
+	const int base_flag = (graph_->mode == DAG_EVAL_VIEWPORT) ?
+		BASE_ENABLED_VIEWPORT : BASE_ENABLED_RENDER;
 	LISTBASE_FOREACH (Base *, base, &view_layer->object_bases) {
-		build_object(base, base->object);
+		const bool is_object_visible = (base->flag & base_flag);
+		if (is_object_visible) {
+			build_object(base, base->object);
+		}
 	}
 
 	build_layer_collections(&view_layer->layer_collections);
diff --git a/source/blender/depsgraph/intern/eval/deg_eval_copy_on_write.cc b/source/blender/depsgraph/intern/eval/deg_eval_copy_on_write.cc
index 573de18e532..3e3bd91a08c 100644
--- a/source/blender/depsgraph/intern/eval/deg_eval_copy_on_write.cc
+++ b/source/blender/depsgraph/intern/eval/deg_eval_copy_on_write.cc
@@ -57,6 +57,7 @@
 #include "BKE_scene.h"
 
 #include "DEG_depsgraph.h"
+#include "DEG_depsgraph_query.h"
 
 #include "MEM_guardedalloc.h"
 
@@ -326,6 +327,71 @@ bool scene_copy_inplace_no_main(const Scene *scene, Scene *new_scene)
 	return result;
 }
 
+/* Remove all view layers but the one which corresponds to an input one. */
+void scene_remove_unused_view_layers(const Depsgraph *depsgraph,
+                                     Scene *scene_cow)
+{
+	ViewLayer *view_layer_input = depsgraph->view_layer;
+	ViewLayer *view_layer_eval = NULL;
+	/* Find evaluated view layer. At the same time we free memory used by
+	 * all other of the view layers. */
+	for (ViewLayer *view_layer_cow =
+	             reinterpret_cast<ViewLayer *>(scene_cow->view_layers.first),
+	               *view_layer_next;
+	     view_layer_cow != NULL;
+	     view_layer_cow = view_layer_next)
+	{
+		view_layer_next = view_layer_cow->next;
+		if (STREQ(view_layer_input->name, view_layer_cow->name)) {
+			view_layer_eval = view_layer_cow;
+		}
+		else {
+			BKE_view_layer_free_ex(view_layer_cow, false);
+		}
+	}
+	BLI_assert(view_layer_eval != NULL);
+	/* Make evaluated view layer the only one in the evaluated scene. */
+	view_layer_eval->prev = view_layer_eval->next = NULL;
+	scene_cow->view_layers.first = view_layer_eval;
+	scene_cow->view_layers.last = view_layer_eval;
+}
+
+/* Makes it so given view layer only has bases corresponding to a visible
+ * objects. */
+void view_layer_remove_invisible_bases(const Depsgraph *depsgraph,
+                                       ViewLayer *view_layer)
+{
+	const int base_visible_flag = (depsgraph->mode == DAG_EVAL_VIEWPORT) ?
+		BASE_ENABLED_VIEWPORT : BASE_ENABLED_RENDER;
+	ListBase visible_bases = {NULL, NULL};
+	for (Base *base = reinterpret_cast<Base *>(view_layer->object_bases.first),
+	          *base_next;
+	     base != NULL;
+	     base = base_next)
+	{
+		base_next = base->next;
+		const bool is_object_visible = (base->flag & base_visible_flag);
+		if (is_object_visible) {
+			BLI_addtail(&visible_bases, base);
+		}
+		else {
+			MEM_freeN(base);
+		}
+	}
+	view_layer->object_bases = visible_bases;
+}
+
+void scene_cleanup_view_layers(const Depsgraph *depsgraph, Scene *scene_cow)
+{
+	scene_remove_unused_view_layers(depsgraph, scene_cow);
+	view_layer_remove_invisible_bases(
+	        depsgraph,
+	        reinterpret_cast<ViewLayer *>(scene_cow->view_layers.first));
+	/* TODO(sergey): Remove objects from collections as well.
+	 * Not a HUGE deal for now, nobody is looking into those CURRENTLY.
+	 * Still not an excuse to have those. */
+}
+
 /* Check whether given ID is expanded or still a shallow copy. */
 BLI_INLINE bool check_datablock_expanded(const ID *id_cow)
 {
@@ -338,7 +404,7 @@ BLI_INLINE bool check_datablock_expanded(const ID *id_cow)
  * TODO(sergey): How to make it more robust for the future, so we don't have
  * to maintain exception lists all over the code?
  */
-static bool check_datablocks_copy_on_writable(const ID *id_orig)
+bool check_datablocks_copy_on_writable(const ID *id_orig)
 {
 	const ID_Type id_type = GS(id_orig->name);
 	/* We shouldn't bother if copied ID is same as original one. */
@@ -643,6 +709,9 @@ ID *deg_expand_copy_on_write_datablock(const Depsgraph *depsgraph,
 		case ID_SCE:
 		{
 			done = scene_copy_inplace_no_main((Scene *)id_orig, (Scene *)id_cow);
+			if (done) {
+				scene_cleanup_view_layers(depsgraph, (Scene *)id_cow);
+			}
 			break;
 		}
 		case ID_ME:
diff --git a/source/blender/depsgraph/intern/nodes/deg_node_id.cc b/source/blender/depsgraph/intern/nodes/deg_node_id.cc
index 808379006af..b194e304e99 100644
--- a/source/blender/depsgraph/intern/nodes/deg_node_id.cc
+++ b/source/blender/depsgraph/intern/nodes/deg_node_id.cc
@@ -106,6 +106,7 @@ void IDDepsNode::init(const ID *id, const char *UNUSED(subdata))
 	previous_eval_flags = 0;
 	linked_state = DEG_ID_LINKED_INDIRECTLY;
 	is_directly_visible = true;
+	is_collection_fully_expanded = false;
 
 	visible_components_mask = 0;
 	previously_visible_components_mask = 0;
diff --git a/source/blender/depsgraph/intern/nodes/deg_node_id.h b/source/blender/depsgraph/intern/nodes/deg_node_id.h
index ed87e7f6ef1..f8d05eddffd 100644
--- a/source/blender/depsgraph/intern/nodes/deg_node_id.h
+++ b/source/blender/depsgraph/intern/nodes/deg_node_id.h
@@ -86,6 +86,10 @@ struct IDDepsNode : public DepsNode {
 	/* Indicates the datablock is visible in the evaluated scene. */
 	bool is_directly_visible;
 
+	/* For the collection type of ID, denotes whether collection was fully
+	 * recursed into. 	 */
+	bool is_collection_fully_expanded;
+
 	IDComponentsMask visible_components_mask;
 	IDComponentsMask previously_visible_components_mask;