Outliner visibility unification: Implement 3 levels of viewport visibility

Now collection and objects can be either:
* Disabled for all the view layers.
* Hidden for a view layer but not necessarily for all others.
* Visible for a view layer but not necessarily for all others.

Regarding icons: Whatever we decide to use for the "Hidden for all view
layers" needs to be a toggle-like icon. Because when viewing "Scenes"
instead of "View Layer" in the outliner we should be able to edit the
collection "Hidden for all the view layers" as an on/off option.

The operators are accessible via a Visibility context menu or shortcuts:
* Ctrl + Click: Isolate collection (use shift to extend).
* Alt + Click: Disable collection.
* Shift + Click: Hide/Show collection and its children (objects and collections)

Things yet to be tackled:
* Object outliner context menu can also get a Visibility sub-menu.
* Get better icons for viewport enable/disable.

Note:
* When using emulate 3 button mouse alt+click is used for 2d panning.
  In this case users have to use the operator from the menu.

See T57857 for discussion.

Patch: https://developer.blender.org/D4011
Reviewers: brecht and sergey

Thanks to the reviewers and William Reynish and Julien Kasper in
particular for the feedback.

2 files changed, 122 insertions, 38 deletions

diff --git a/source/blender/blenkernel/BKE_layer.h b/source/blender/blenkernel/BKE_layer.h
index cff80ef3467..65c0202ef66 100644
--- a/source/blender/blenkernel/BKE_layer.h
+++ b/source/blender/blenkernel/BKE_layer.h
@@ -109,7 +109,8 @@ bool BKE_layer_collection_has_layer_collection(
         struct LayerCollection *lc_parent, struct LayerCollection *lc_child);
 
 void BKE_base_set_visible(struct Scene *scene, struct ViewLayer *view_layer, struct Base *base, bool extend);
-void BKE_layer_collection_set_visible(struct Scene *scene, struct ViewLayer *view_layer, struct LayerCollection *lc, bool extend);
+bool BKE_layer_collection_isolate(struct Scene *scene, struct ViewLayer *view_layer, struct LayerCollection *lc, bool extend);
+bool BKE_layer_collection_set_visible(struct ViewLayer *view_layer, struct LayerCollection *lc, const bool visible, const bool hierarchy);
 
 /* evaluation */
 
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index 0b49a389ab1..f9d39c840e2 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -669,7 +669,9 @@ static short layer_collection_sync(
 			lc->runtime_flag = child_runtime_flag;
 		}
 
-		if ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0) {
+		if (((child_restrict & COLLECTION_RESTRICT_VIEW) == 0) &&
+		    ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0))
+		{
 			lc->runtime_flag |= LAYER_COLLECTION_VISIBLE;
 		}
 
@@ -701,15 +703,18 @@ static short layer_collection_sync(
 			int object_restrict = base->object->restrictflag;
 
 			if (((child_restrict & COLLECTION_RESTRICT_VIEW) == 0) &&
-			    ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0) &&
 			    ((object_restrict & OB_RESTRICT_VIEW) == 0))
 			{
-				base->flag |= BASE_VISIBLE | BASE_ENABLED | BASE_ENABLED_VIEWPORT;
+				base->flag |= BASE_ENABLED | BASE_ENABLED_VIEWPORT;
 
-				if (((child_restrict & COLLECTION_RESTRICT_SELECT) == 0) &&
-				    ((object_restrict & OB_RESTRICT_SELECT) == 0))
-				{
-					base->flag |= BASE_SELECTABLE;
+				if ((child_layer_restrict & LAYER_COLLECTION_RESTRICT_VIEW) == 0) {
+					base->flag |= BASE_VISIBLE;
+
+					if (((child_restrict & COLLECTION_RESTRICT_SELECT) == 0) &&
+						((object_restrict & OB_RESTRICT_SELECT) == 0))
+					{
+						base->flag |= BASE_SELECTABLE;
+					}
 				}
 			}
 
@@ -720,7 +725,7 @@ static short layer_collection_sync(
 			}
 
 			/* Update runtime flags used for display and tools. */
-			if (base->flag & BASE_VISIBLE) {
+			if (base->flag & BASE_ENABLED) {
 				lc->runtime_flag |= LAYER_COLLECTION_HAS_ENABLED_OBJECTS;
 			}
 
@@ -987,48 +992,124 @@ static void layer_collection_flag_unset_recursive(LayerCollection *lc, const int
 }
 
 /**
- * Set collection per-view layer visiblity.
- * When not extending, we show all the direct parents and all children of the layer collection.
+ * Return true if something changed. */
+static bool layer_collection_collection_flag_unset_recursive(LayerCollection *lc, const int flag)
+{
+	bool changed = (lc->collection->flag & flag) != 0;
+
+	lc->collection->flag &= ~flag;
+	for (LayerCollection *lc_iter = lc->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		changed |= layer_collection_collection_flag_unset_recursive(lc_iter, flag);
+	}
+
+	return changed;
+}
+
+/**
+ * Isolate the collection - hide all other collections but this one.
+ * Make sure to show all the direct parents and all children of the layer collection as well.
+ * When extending we simply show the collections and its direct family.
+ *
+ * Return whether depsgraph needs update.
  */
-void BKE_layer_collection_set_visible(Scene *scene, ViewLayer *view_layer, LayerCollection *lc, bool extend)
+bool BKE_layer_collection_isolate(Scene *scene, ViewLayer *view_layer, LayerCollection *lc, bool extend)
 {
+	bool depsgraph_need_update = false;
 	if (!extend) {
-		/* Make only this collection visible. */
+		/* Hide all collections . */
 		for (LayerCollection *lc_iter = view_layer->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
 			layer_collection_flag_set_recursive(lc_iter, LAYER_COLLECTION_RESTRICT_VIEW);
 		}
+	}
 
-		/* Make all the direct parents visible. */
-		LayerCollection *lc_parent = lc;
-		LayerCollection *lc_master = view_layer->layer_collections.first;
-		for (LayerCollection *lc_iter = lc_master->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+	/* Make all the direct parents visible. */
+	LayerCollection *lc_parent = lc;
+	LayerCollection *lc_master = view_layer->layer_collections.first;
+	for (LayerCollection *lc_iter = lc_master->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		if (BKE_layer_collection_has_layer_collection(lc_iter, lc)) {
+			lc_parent = lc_iter;
+			break;
+		}
+	}
+
+	while (lc_parent != lc) {
+		depsgraph_need_update |= (lc_parent->collection->flag & COLLECTION_RESTRICT_VIEW) != 0;
+		lc_parent->collection->flag &= ~COLLECTION_RESTRICT_VIEW;
+		lc_parent->flag &= ~LAYER_COLLECTION_RESTRICT_VIEW;
+
+		for (LayerCollection *lc_iter = lc_parent->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
 			if (BKE_layer_collection_has_layer_collection(lc_iter, lc)) {
 				lc_parent = lc_iter;
 				break;
 			}
 		}
+	}
 
-		while (lc_parent != lc) {
-			lc_parent->flag &= ~LAYER_COLLECTION_RESTRICT_VIEW;
+	/* Make all the children visible. */
+	layer_collection_flag_unset_recursive(lc, LAYER_COLLECTION_RESTRICT_VIEW);
+	depsgraph_need_update |= layer_collection_collection_flag_unset_recursive(lc, COLLECTION_RESTRICT_VIEW);
 
-			for (LayerCollection *lc_iter = lc_parent->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
-				if (BKE_layer_collection_has_layer_collection(lc_iter, lc)) {
-					lc_parent = lc_iter;
-					break;
-				}
-			}
-		}
+	BKE_layer_collection_activate(view_layer, lc);
+
+	BKE_layer_collection_sync(scene, view_layer);
+
+	return depsgraph_need_update;
+}
 
-		/* Make all the children visible. */
-		layer_collection_flag_unset_recursive(lc, LAYER_COLLECTION_RESTRICT_VIEW);
+static void layer_collection_bases_show_recursive(ViewLayer *view_layer, LayerCollection *lc)
+{
+	for (CollectionObject *cob = lc->collection->gobject.first; cob; cob = cob->next) {
+		Base *base = BKE_view_layer_base_find(view_layer, cob->ob);
+		base->flag &= ~BASE_HIDDEN;
+		base->object->restrictflag &= ~OB_RESTRICT_VIEW;
+	}
+	for (LayerCollection *lc_iter = lc->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		layer_collection_bases_show_recursive(view_layer, lc_iter);
+	}
+}
 
-		BKE_layer_collection_activate(view_layer, lc);
+static void layer_collection_bases_hide_recursive(ViewLayer *view_layer, LayerCollection *lc)
+{
+	for (CollectionObject *cob = lc->collection->gobject.first; cob; cob = cob->next) {
+		Base *base = BKE_view_layer_base_find(view_layer, cob->ob);
+		base->flag |= BASE_HIDDEN;
 	}
-	else {
-		lc->flag ^= LAYER_COLLECTION_RESTRICT_VIEW;
+	for (LayerCollection *lc_iter = lc->layer_collections.first; lc_iter; lc_iter = lc_iter->next) {
+		layer_collection_bases_hide_recursive(view_layer, lc_iter);
 	}
+}
 
-	BKE_layer_collection_sync(scene, view_layer);
+/**
+ * Hide/show all the elements of a collection.
+ * Enable a disable collection if needs be.
+ *
+ * Return true if depsgraph needs update.
+ */
+bool BKE_layer_collection_set_visible(ViewLayer *view_layer, LayerCollection *lc, const bool visible, const bool hierarchy)
+{
+	bool depsgraph_changed = false;
+	if (hierarchy) {
+		if (visible) {
+			depsgraph_changed |= layer_collection_collection_flag_unset_recursive(lc, COLLECTION_RESTRICT_VIEW);
+			layer_collection_flag_unset_recursive(lc, LAYER_COLLECTION_RESTRICT_VIEW);
+			layer_collection_bases_show_recursive(view_layer, lc);
+		}
+		else {
+			layer_collection_flag_set_recursive(lc, LAYER_COLLECTION_RESTRICT_VIEW);
+			layer_collection_bases_hide_recursive(view_layer, lc);
+		}
+	}
+	else {
+		if (visible) {
+			depsgraph_changed |= (lc->collection->flag & COLLECTION_RESTRICT_VIEW) != 0;
+			lc->flag &= ~LAYER_COLLECTION_RESTRICT_VIEW;
+			lc->collection->flag &= ~COLLECTION_RESTRICT_VIEW;
+		}
+		else {
+			lc->flag |= LAYER_COLLECTION_RESTRICT_VIEW;
+		}
+	}
+	return depsgraph_changed;
 }
 
 /* ---------------------------------------------------------------------- */
@@ -1384,7 +1465,7 @@ void BKE_layer_eval_view_layer(
 
 	/* Visibility based on depsgraph mode. */
 	const eEvaluationMode mode = DEG_get_mode(depsgraph);
-	const int base_visible_flag = (mode == DAG_EVAL_VIEWPORT)
+	const int base_enabled_flag = (mode == DAG_EVAL_VIEWPORT)
 	        ? BASE_ENABLED_VIEWPORT
 	        : BASE_ENABLED_RENDER;
 	/* Create array of bases, for fast index-based lookup. */
@@ -1395,10 +1476,12 @@ void BKE_layer_eval_view_layer(
 	int base_index = 0;
 	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
 		/* Compute visibility for depsgraph evaluation mode. */
-		if (base->flag & base_visible_flag) {
-			base->flag |= BASE_ENABLED | BASE_VISIBLE;
-			if (mode == DAG_EVAL_VIEWPORT && (base->flag & BASE_HIDDEN)) {
-				base->flag &= ~BASE_VISIBLE;
+		if (base->flag & base_enabled_flag) {
+			base->flag |= BASE_ENABLED;
+
+			/* When rendering, visibility is controlled by the enable/disable option. */
+			if (mode == DAG_EVAL_RENDER) {
+				base->flag |= BASE_VISIBLE;
 			}
 		}
 		else {
@@ -1416,7 +1499,7 @@ void BKE_layer_eval_view_layer(
 		Base *base_orig = view_layer_orig->object_bases.first;
 		const Base *base_eval = view_layer->object_bases.first;
 		while (base_orig != NULL) {
-			if (base_orig->flag & base_visible_flag) {
+			if (base_orig->flag & base_enabled_flag) {
 				base_orig->flag = base_eval->flag;
 				base_eval = base_eval->next;
 			}