Per view-layer collection visibility

We still control this in the viewport collections visibility menu. But
now we are actually changing the visibility of the collections, not of
the objects.

If a collection is indirectly invisible (because one of its parents are
invisible) we gray it out.

Also if you click directly in the collection names, it "isolates" the
collection by hiding all collections, and showing the direct parents and
all the children of the selected collection.

Development Note:

Right now I'm excluding the hidden collections from the depsgraph.
Thus the need for tagging relations to update.

If this proves to be too slow, we can change.

9 files changed, 118 insertions, 47 deletions

diff --git a/source/blender/blenkernel/BKE_layer.h b/source/blender/blenkernel/BKE_layer.h
index e98fa189379..5e0d3f6c5a5 100644
--- a/source/blender/blenkernel/BKE_layer.h
+++ b/source/blender/blenkernel/BKE_layer.h
@@ -113,6 +113,8 @@ bool BKE_layer_collection_objects_select(
         struct ViewLayer *view_layer, struct LayerCollection *lc, bool deselect);
 bool BKE_layer_collection_has_selected_objects(
         struct ViewLayer *view_layer, struct LayerCollection *lc);
+bool BKE_layer_collection_has_layer_collection(
+        struct LayerCollection *lc_parent, struct LayerCollection *lc_child);
 
 void BKE_base_set_visible(struct Scene *scene, struct ViewLayer *view_layer, struct Base *base, bool extend);
 void BKE_layer_collection_set_visible(struct Scene *scene, struct ViewLayer *view_layer, struct LayerCollection *lc, bool extend);
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index 9931c29a8c2..badceb5c01d 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -610,7 +610,7 @@ int BKE_layer_collection_findindex(ViewLayer *view_layer, const LayerCollection
 static short layer_collection_sync(
         ViewLayer *view_layer, const ListBase *lb_scene,
         ListBase *lb_layer, ListBase *new_object_bases,
-        short parent_exclude, short parent_restrict)
+        short parent_exclude, short parent_restrict, short parent_layer_restrict)
 {
 	/* TODO: support recovery after removal of intermediate collections, reordering, ..
 	 * For local edits we can make editing operating do the appropriate thing, but for
@@ -655,15 +655,17 @@ static short layer_collection_sync(
 
 		/* Collection restrict is inherited. */
 		short child_restrict = parent_restrict;
+		short child_layer_restrict = parent_layer_restrict;
 		if (!(collection->flag & COLLECTION_IS_MASTER)) {
 			child_restrict |= collection->flag;
+			child_layer_restrict |= lc->flag;
 		}
 
 		/* Sync child collections. */
 		short child_runtime_flag = layer_collection_sync(
 		        view_layer, &collection->children,
 		        &lc->layer_collections, new_object_bases,
-		        lc->flag, child_restrict);
+		        lc->flag, child_restrict, child_layer_restrict);
 
 		/* Layer collection exclude is not inherited. */
 		if (lc->flag & LAYER_COLLECTION_EXCLUDE) {
@@ -674,6 +676,10 @@ static short layer_collection_sync(
 			lc->runtime_flag = child_runtime_flag;
 		}
 
+		if ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0) {
+			lc->runtime_flag |= LAYER_COLLECTION_VISIBLE;
+		}
+
 		/* Sync objects, except if collection was excluded. */
 		for (CollectionObject *cob = collection->gobject.first; cob; cob = cob->next) {
 			if (cob->ob == NULL) {
@@ -701,6 +707,7 @@ static short layer_collection_sync(
 			int object_restrict = base->object->restrictflag;
 
 			if (((child_restrict & COLLECTION_RESTRICT_VIEW) == 0) &&
+			    ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0) &&
 			    ((object_restrict & OB_RESTRICT_VIEW) == 0))
 			{
 				base->flag |= BASE_VISIBLE | BASE_ENABLED | BASE_ENABLED_VIEWPORT;
@@ -714,7 +721,6 @@ static short layer_collection_sync(
 
 			if (((child_restrict & COLLECTION_RESTRICT_RENDER) == 0) &&
 			    ((object_restrict & OB_RESTRICT_RENDER) == 0))
-
 			{
 				base->flag |= BASE_ENABLED_RENDER;
 			}
@@ -791,11 +797,11 @@ void BKE_layer_collection_sync(const Scene *scene, ViewLayer *view_layer)
 	const ListBase collections = {&child, &child};
 	ListBase new_object_bases = {NULL, NULL};
 
-	const short parent_exclude = 0, parent_restrict = 0;
+	const short parent_exclude = 0, parent_restrict = 0, parent_layer_restrict = 0;
 	layer_collection_sync(
 	        view_layer, &collections,
 	        &view_layer->layer_collections, &new_object_bases,
-	        parent_exclude, parent_restrict);
+	        parent_exclude, parent_restrict, parent_layer_restrict);
 
 	/* Any remaning object bases are to be removed. */
 	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
@@ -919,7 +925,7 @@ bool BKE_layer_collection_has_selected_objects(ViewLayer *view_layer, LayerColle
 		for (CollectionObject *cob = lc->collection->gobject.first; cob; cob = cob->next) {
 			Base *base = BKE_view_layer_base_find(view_layer, cob->ob);
 
-			if (base && (base->flag & BASE_SELECTED)) {
+			if (base && (base->flag & BASE_SELECTED) && (base->flag & BASE_VISIBLE)) {
 				return true;
 			}
 		}
@@ -934,6 +940,20 @@ bool BKE_layer_collection_has_selected_objects(ViewLayer *view_layer, LayerColle
 	return false;
 }
 
+bool BKE_layer_collection_has_layer_collection(LayerCollection *lc_parent, LayerCollection *lc_child)
+{
+	if (lc_parent == lc_child) {
+		return true;
+	}
+
+	for (LayerCollection *lc_iter = lc_parent->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		if (BKE_layer_collection_has_layer_collection(lc_iter, lc_child)) {
+			return true;
+		}
+	}
+	return false;
+}
+
 /* ---------------------------------------------------------------------- */
 
 /* Update after toggling visibility of an object base. */
@@ -955,44 +975,62 @@ void BKE_base_set_visible(Scene *scene, ViewLayer *view_layer, Base *base, bool
 	BKE_layer_collection_sync(scene, view_layer);
 }
 
+static void layer_collection_flag_set_recursive(LayerCollection *lc, const int flag)
+{
+	lc->flag |= flag;
+	for (LayerCollection *lc_iter = lc->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		layer_collection_flag_set_recursive(lc_iter, flag);
+	}
+}
+
+static void layer_collection_flag_unset_recursive(LayerCollection *lc, const int flag)
+{
+	lc->flag &= ~flag;
+	for (LayerCollection *lc_iter = lc->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		layer_collection_flag_unset_recursive(lc_iter, flag);
+	}
+}
+
+/**
+ * Set collection per-view layer visiblity.
+ * When not extending, we show all the direct parents and all children of the layer collection.
+ */
 void BKE_layer_collection_set_visible(Scene *scene, ViewLayer *view_layer, LayerCollection *lc, bool extend)
 {
 	if (!extend) {
-		/* Make only objects from one collection visible. */
-		for (Base *base = view_layer->object_bases.first; base; base = base->next) {
-			base->flag |= BASE_HIDDEN;
+		/* Make only this collection visible. */
+		for (LayerCollection *lc_iter = view_layer->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+			layer_collection_flag_set_recursive(lc_iter, LAYER_COLLECTION_RESTRICT_VIEW);
 		}
 
-		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(lc->collection, ob)
-		{
-			Base *base = BLI_ghash_lookup(view_layer->object_bases_hash, ob);
-
-			if (base) {
-				base->flag &= ~BASE_HIDDEN;
+		/* Make all the direct parents visible. */
+		LayerCollection *lc_parent = lc;
+		LayerCollection *lc_master = view_layer->layer_collections.first;
+		for (LayerCollection *lc_iter = lc_master->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+			if (BKE_layer_collection_has_layer_collection(lc_iter, lc)) {
+				lc_parent = lc_iter;
+				break;
 			}
 		}
-		FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
 
-		BKE_layer_collection_activate(view_layer, lc);
-	}
-	else {
-		/* Toggle visibility of objects from collection. */
-		bool hide = (lc->runtime_flag & LAYER_COLLECTION_HAS_VISIBLE_OBJECTS) != 0;
-
-		FOREACH_COLLECTION_OBJECT_RECURSIVE_BEGIN(lc->collection, ob)
-		{
-			Base *base = BLI_ghash_lookup(view_layer->object_bases_hash, ob);
+		while (lc_parent != lc) {
+			lc_parent->flag &= ~LAYER_COLLECTION_RESTRICT_VIEW;
 
-			if (base) {
-				if (hide) {
-					base->flag |= BASE_HIDDEN;
-				}
-				else {
-					base->flag &= ~BASE_HIDDEN;
+			for (LayerCollection *lc_iter = lc_parent->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+				if (BKE_layer_collection_has_layer_collection(lc_iter, lc)) {
+					lc_parent = lc_iter;
+					break;
 				}
 			}
 		}
-		FOREACH_COLLECTION_OBJECT_RECURSIVE_END;
+
+		/* Make all the children visible. */
+		layer_collection_flag_unset_recursive(lc, LAYER_COLLECTION_RESTRICT_VIEW);
+
+		BKE_layer_collection_activate(view_layer, lc);
+	}
+	else {
+		lc->flag ^= LAYER_COLLECTION_RESTRICT_VIEW;
 	}
 
 	BKE_layer_collection_sync(scene, view_layer);
@@ -1101,7 +1139,7 @@ typedef struct LayerObjectBaseIteratorData {
 	Base *base;
 } LayerObjectBaseIteratorData;
 
-static bool object_bases_iterator_is_valid_ex(View3D *v3d, Base *base, const int flag)
+static bool object_bases_iterator_is_valid_ex(View3D *v3d, Base *base, const int flags)
 {
 	if (v3d != NULL) {
 		if ((v3d->object_type_exclude_viewport & (1 << base->object->type)) != 0) {
@@ -1113,11 +1151,8 @@ static bool object_bases_iterator_is_valid_ex(View3D *v3d, Base *base, const int
 		}
 	}
 
-	if ((base->flag & flag) == 0) {
-		return false;
-	}
-
-	return true;
+	/* flags may be more than one flag, so we can't check != 0. */
+	return ((base->flag & flags) == flags);
 }
 
 static bool object_bases_iterator_is_valid(View3D *v3d, Base *base)
@@ -1205,12 +1240,12 @@ static void objects_iterator_end(BLI_Iterator *iter)
 
 void BKE_view_layer_selected_objects_iterator_begin(BLI_Iterator *iter, void *data_in)
 {
-	objects_iterator_begin(iter, data_in, BASE_SELECTED);
+	objects_iterator_begin(iter, data_in, BASE_VISIBLE | BASE_SELECTED);
 }
 
 void BKE_view_layer_selected_objects_iterator_next(BLI_Iterator *iter)
 {
-	objects_iterator_next(iter, BASE_SELECTED);
+	objects_iterator_next(iter, BASE_VISIBLE | BASE_SELECTED);
 }
 
 void BKE_view_layer_selected_objects_iterator_end(BLI_Iterator *iter)
@@ -1247,7 +1282,7 @@ void BKE_view_layer_visible_objects_iterator_end(BLI_Iterator *iter)
 
 void BKE_view_layer_selected_editable_objects_iterator_begin(BLI_Iterator *iter, void *data_in)
 {
-	objects_iterator_begin(iter, data_in, BASE_SELECTED);
+	objects_iterator_begin(iter, data_in, BASE_VISIBLE | BASE_SELECTED);
 	if (iter->valid) {
 		if (BKE_object_is_libdata((Object *)iter->current) == false) {
 			// First object is valid (selectable and not libdata) -> all good.
@@ -1264,7 +1299,7 @@ void BKE_view_layer_selected_editable_objects_iterator_next(BLI_Iterator *iter)
 {
 	// Search while there are objects and the one we have is not editable (editable = not libdata).
 	do {
-		objects_iterator_next(iter, BASE_SELECTED);
+		objects_iterator_next(iter, BASE_VISIBLE | BASE_SELECTED);
 	} while (iter->valid && BKE_object_is_libdata((Object *)iter->current) != false);
 }
 
@@ -1281,12 +1316,12 @@ void BKE_view_layer_selected_editable_objects_iterator_end(BLI_Iterator *iter)
 
 void BKE_view_layer_selected_bases_iterator_begin(BLI_Iterator *iter, void *data_in)
 {
-	objects_iterator_begin(iter, data_in, BASE_SELECTED);
+	objects_iterator_begin(iter, data_in, BASE_VISIBLE | BASE_SELECTED);
 }
 
 void BKE_view_layer_selected_bases_iterator_next(BLI_Iterator *iter)
 {
-	object_bases_iterator_next(iter, BASE_SELECTED);
+	object_bases_iterator_next(iter, BASE_VISIBLE | BASE_SELECTED);
 }
 
 void BKE_view_layer_selected_bases_iterator_end(BLI_Iterator *iter)
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
index 70bd533647c..2fff8bae2b2 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes_view_layer.cc
@@ -71,6 +71,11 @@ void DepsgraphNodeBuilder::build_layer_collections(ListBase *lb)
 		COLLECTION_RESTRICT_VIEW : COLLECTION_RESTRICT_RENDER;
 
 	for (LayerCollection *lc = (LayerCollection *)lb->first; lc; lc = lc->next) {
+		if ((graph_->mode == DAG_EVAL_VIEWPORT) &&
+		    ((lc->flag & LAYER_COLLECTION_RESTRICT_VIEW) != 0))
+		{
+			continue;
+		}
 		if (lc->collection->flag & restrict_flag) {
 			continue;
 		}
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
index 999508cffed..6c85803dabe 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations_view_layer.cc
@@ -75,6 +75,11 @@ void DepsgraphRelationBuilder::build_layer_collections(ListBase *lb)
 		COLLECTION_RESTRICT_VIEW : COLLECTION_RESTRICT_RENDER;
 
 	for (LayerCollection *lc = (LayerCollection *)lb->first; lc; lc = lc->next) {
+		if ((graph_->mode == DAG_EVAL_VIEWPORT) &&
+		    ((lc->flag & LAYER_COLLECTION_RESTRICT_VIEW) != 0))
+		{
+			continue;
+		}
 		if ((lc->collection->flag & restrict_flag)) {
 			continue;
 		}
diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index 5852fcff7e9..df1e67efb2d 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -295,6 +295,7 @@ static int object_hide_collection_exec(bContext *C, wmOperator *op)
 
 	BKE_layer_collection_set_visible(scene, view_layer, lc, extend);
 
+	DEG_relations_tag_update(CTX_data_main(C));
 	DEG_id_tag_update(&scene->id, DEG_TAG_BASE_FLAGS_UPDATE);
 	WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, scene);
 
@@ -368,7 +369,7 @@ static int object_hide_collection_invoke(bContext *C, wmOperator *op, const wmEv
 void OBJECT_OT_hide_collection(wmOperatorType *ot)
 {
 	/* identifiers */
-	ot->name = "Hide Objects By Collection";
+	ot->name = "Hide Collection";
 	ot->description = "Show only objects in collection (Shift to extend)";
 	ot->idname = "OBJECT_OT_hide_collection";
 
diff --git a/source/blender/editors/screen/screen_context.c b/source/blender/editors/screen/screen_context.c
index cb7d4f1fee9..df7eb07c4d0 100644
--- a/source/blender/editors/screen/screen_context.c
+++ b/source/blender/editors/screen/screen_context.c
@@ -203,6 +203,9 @@ int ed_screen_context(const bContext *C, const char *member, bContextDataResult
 			if (v3d && ((v3d->object_type_exclude_viewport & (1 << base->object->type)) != 0)) {
 				continue;
 			}
+			if ((base->flag & BASE_VISIBLE) == 0) {
+				continue;
+			}
 			if ((base->flag & BASE_SELECTED) != 0) {
 				CTX_data_list_add(result, &scene->id, &RNA_ObjectBase, base);
 			}
@@ -218,6 +221,9 @@ int ed_screen_context(const bContext *C, const char *member, bContextDataResult
 			if (v3d && ((v3d->object_type_exclude_viewport & (1 << base->object->type)) != 0)) {
 				continue;
 			}
+			if ((base->flag & BASE_VISIBLE) == 0) {
+				continue;
+			}
 			if ((base->flag & BASE_SELECTED) != 0) {
 				if (0 == BKE_object_is_libdata(base->object)) {
 					CTX_data_list_add(result, &scene->id, &RNA_ObjectBase, base);
diff --git a/source/blender/editors/transform/transform_conversions.c b/source/blender/editors/transform/transform_conversions.c
index 0fca84074f5..47bde2a9af7 100644
--- a/source/blender/editors/transform/transform_conversions.c
+++ b/source/blender/editors/transform/transform_conversions.c
@@ -5906,7 +5906,8 @@ static int count_proportional_objects(TransInfo *t)
 		for (Base *base = view_layer->object_bases.first; base; base = base->next) {
 			/* all base not already selected or marked that is editable */
 			if ((base->object->flag & (BA_TRANSFORM_CHILD | BA_TRANSFORM_PARENT)) == 0 &&
-			    (base->flag & BASE_SELECTED) == 0 &&
+			    ((base->flag & BASE_VISIBLE) != 0 &&
+			     (base->flag & BASE_SELECTED) == 0) &&
 			    (BASE_EDITABLE_BGMODE(v3d, base)))
 			{
 				mark_children(base->object);
@@ -5920,7 +5921,8 @@ static int count_proportional_objects(TransInfo *t)
 		 * selection and it is editable.
 		 */
 		if ((ob->flag & (BA_TRANSFORM_CHILD | BA_TRANSFORM_PARENT)) == 0 &&
-		    (base->flag & BASE_SELECTED) == 0 &&
+		    ((base->flag & BASE_VISIBLE) != 0 &&
+		     (base->flag & BASE_SELECTED) == 0) &&
 		    (BASE_EDITABLE_BGMODE(v3d, base)))
 		{
 			flush_trans_object_base_deps_flag(depsgraph, ob);
diff --git a/source/blender/makesdna/DNA_layer_types.h b/source/blender/makesdna/DNA_layer_types.h
index c3e819c5538..8cc305d9203 100644
--- a/source/blender/makesdna/DNA_layer_types.h
+++ b/source/blender/makesdna/DNA_layer_types.h
@@ -117,6 +117,7 @@ enum {
 	LAYER_COLLECTION_EXCLUDE = (1 << 4),
 	LAYER_COLLECTION_HOLDOUT = (1 << 5),
 	LAYER_COLLECTION_INDIRECT_ONLY = (1 << 6),
+	LAYER_COLLECTION_RESTRICT_VIEW = (1 << 7),
 };
 
 /* Layer Collection->runtime_flag */
@@ -125,6 +126,7 @@ enum {
 	LAYER_COLLECTION_HAS_VISIBLE_OBJECTS = (1 << 1),
 	LAYER_COLLECTION_HAS_HIDDEN_OBJECTS = (1 << 2),
 	LAYER_COLLECTION_HAS_ENABLED_OBJECTS = (1 << 3),
+	LAYER_COLLECTION_VISIBLE = (1 << 4),
 };
 
 /* ViewLayer->flag */
diff --git a/source/blender/makesrna/intern/rna_layer.c b/source/blender/makesrna/intern/rna_layer.c
index 46d1f5bef7d..cdb87169a45 100644
--- a/source/blender/makesrna/intern/rna_layer.c
+++ b/source/blender/makesrna/intern/rna_layer.c
@@ -297,6 +297,19 @@ static void rna_def_layer_collection(BlenderRNA *brna)
 	                         "in the view layer");
 	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
 
+	prop = RNA_def_property(srna, "hide_viewport", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_RESTRICT_VIEW);
+	RNA_def_property_override_flag(prop, PROPOVERRIDE_OVERRIDABLE_STATIC);
+	RNA_def_property_ui_icon(prop, ICON_HIDE_OFF, -1);
+	RNA_def_property_ui_text(prop, "Disable Viewport", "Disable collection in viewport for this view layer");
+	RNA_def_property_update(prop, NC_SCENE | ND_LAYER_CONTENT, "rna_LayerCollection_use_update");
+
+	prop = RNA_def_property(srna, "is_visible", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "runtime_flag", LAYER_COLLECTION_VISIBLE);
+	RNA_def_property_clear_flag(prop, PROP_EDITABLE);
+	RNA_def_property_ui_text(prop, "Visible",
+	                         "Whether this collection is visible, take into account the collection parent");
+
 	func = RNA_def_function(srna, "has_objects", "rna_LayerCollection_has_objects");
 	RNA_def_function_ui_description(func, "");
 	RNA_def_function_return(func, RNA_def_boolean(func, "result", 0, "", ""));