Group collection viewport/render options and remove collection visibility

Users can change the group collection visibility in the outliner
when looking at groups.

Regular collections on the other hand don't have any special visibility control,
if you need a collection to be invisible during render, either don't link it
into the view layer used for F12, or disable it.

This includes:
* Updated unittests - update your lib/tests/layers folder.
* Subversion bump - branches be aware of that.

Note:
Although we are using eval_ctx to determine the visibility of a group collection
when rendering, the depsgraph is still using the same depsgraph for the viewport
and the render engine, so at the moment the render visibility is ignored.

Following next is a workaround for this separately to tag the groups before and
after rendering to tackle that.

3 files changed, 24 insertions, 35 deletions

diff --git a/source/blender/blenkernel/BKE_blender_version.h b/source/blender/blenkernel/BKE_blender_version.h
index 8d326c6441c..27223afe4ff 100644
--- a/source/blender/blenkernel/BKE_blender_version.h
+++ b/source/blender/blenkernel/BKE_blender_version.h
@@ -28,7 +28,7 @@
  * and keep comment above the defines.
  * Use STRINGIFY() rather than defining with quotes */
 #define BLENDER_VERSION         280
-#define BLENDER_SUBVERSION      2
+#define BLENDER_SUBVERSION      3
 /* Several breakages with 270, e.g. constraint deg vs rad */
 #define BLENDER_MINVERSION      270
 #define BLENDER_MINSUBVERSION   6
diff --git a/source/blender/blenkernel/BKE_layer.h b/source/blender/blenkernel/BKE_layer.h
index e4f8b8790f6..6a762c9b711 100644
--- a/source/blender/blenkernel/BKE_layer.h
+++ b/source/blender/blenkernel/BKE_layer.h
@@ -102,7 +102,6 @@ struct LayerCollection *BKE_collection_link(struct ViewLayer *view_layer, struct
 void BKE_collection_unlink(struct ViewLayer *view_layer, struct LayerCollection *lc);
 
 void BKE_collection_enable(struct ViewLayer *view_layer, struct LayerCollection *lc);
-void BKE_collection_disable(struct ViewLayer *view_layer, struct LayerCollection *lc);
 
 bool BKE_view_layer_has_collection(struct ViewLayer *view_layer, const struct SceneCollection *sc);
 bool BKE_scene_has_object(struct Scene *scene, struct Object *ob);
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index 84eb4ee2e33..2e6cd769db8 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -42,6 +42,8 @@
 #include "BKE_node.h"
 #include "BKE_workspace.h"
 
+#include "DEG_depsgraph.h"
+
 #include "DNA_group_types.h"
 #include "DNA_ID.h"
 #include "DNA_layer_types.h"
@@ -1018,7 +1020,8 @@ static void layer_collection_enable(ViewLayer *view_layer, LayerCollection *lc)
 /**
  * Enable collection
  * Add its objects bases to ViewLayer
- * Depsgraph needs to be rebuilt afterwards
+ *
+ * Only around for doversion.
  */
 void BKE_collection_enable(ViewLayer *view_layer, LayerCollection *lc)
 {
@@ -1030,33 +1033,6 @@ void BKE_collection_enable(ViewLayer *view_layer, LayerCollection *lc)
 	layer_collection_enable(view_layer, lc);
 }
 
-/**
- * Recursively disable nested collections
- */
-static void layer_collection_disable(ViewLayer *view_layer, LayerCollection *lc)
-{
-	layer_collection_objects_unpopulate(view_layer, lc);
-
-	for (LayerCollection *nlc = lc->layer_collections.first; nlc; nlc = nlc->next) {
-		layer_collection_disable(view_layer, nlc);
-	}
-}
-
-/**
- * Disable collection
- * Remove all its object bases from ViewLayer
- * Depsgraph needs to be rebuilt afterwards
- */
-void BKE_collection_disable(ViewLayer *view_layer, LayerCollection *lc)
-{
-	if ((lc->flag & COLLECTION_DISABLED) != 0) {
-		return;
-	}
-
-	lc->flag |= COLLECTION_DISABLED;
-	layer_collection_disable(view_layer, lc);
-}
-
 static void layer_collection_object_add(ViewLayer *view_layer, LayerCollection *lc, Object *ob)
 {
 	Base *base = object_base_add(view_layer, ob);
@@ -1068,7 +1044,7 @@ static void layer_collection_object_add(ViewLayer *view_layer, LayerCollection *
 		return;
 	}
 
-	bool is_visible = (lc->flag & COLLECTION_VISIBLE) != 0;
+	bool is_visible = ((lc->flag & COLLECTION_VIEWPORT) != 0) && ((lc->flag & COLLECTION_DISABLED) == 0);
 	bool is_selectable = is_visible && ((lc->flag & COLLECTION_SELECTABLE) != 0);
 
 	if (is_visible) {
@@ -1117,7 +1093,7 @@ static LayerCollection *layer_collection_add(ViewLayer *view_layer, LayerCollect
 	LayerCollection *lc = MEM_callocN(sizeof(LayerCollection), "Collection Base");
 
 	lc->scene_collection = sc;
-	lc->flag = COLLECTION_VISIBLE | COLLECTION_SELECTABLE;
+	lc->flag = COLLECTION_SELECTABLE | COLLECTION_VIEWPORT | COLLECTION_RENDER;
 
 	lc->properties = IDP_New(IDP_GROUP, &val, ROOT_PROP);
 	collection_engine_settings_init(lc->properties, false);
@@ -2118,7 +2094,21 @@ static const char *collection_type_lookup[] =
     "Group Internal", /* COLLECTION_TYPE_GROUP_INTERNAL */
 };
 
-void BKE_layer_eval_layer_collection(const struct EvaluationContext *UNUSED(eval_ctx),
+static bool layer_collection_visible_get(const EvaluationContext *eval_ctx, LayerCollection *layer_collection)
+{
+	bool is_visible = (layer_collection->flag & COLLECTION_DISABLED) == 0;
+
+	if (eval_ctx->mode == DAG_EVAL_VIEWPORT) {
+		is_visible &= (layer_collection->flag & COLLECTION_VIEWPORT) != 0;
+	}
+	else {
+		is_visible &= (layer_collection->flag & COLLECTION_RENDER) != 0;
+	}
+
+	return is_visible;
+}
+
+void BKE_layer_eval_layer_collection(const EvaluationContext *eval_ctx,
                                      LayerCollection *layer_collection,
                                      LayerCollection *parent_layer_collection)
 {
@@ -2134,11 +2124,11 @@ void BKE_layer_eval_layer_collection(const struct EvaluationContext *UNUSED(eval
 
 	/* visibility */
 	layer_collection->flag_evaluated = layer_collection->flag;
-	bool is_visible = (layer_collection->flag & COLLECTION_VISIBLE) != 0;
+	bool is_visible = layer_collection_visible_get(eval_ctx, layer_collection);
 	bool is_selectable = is_visible && ((layer_collection->flag & COLLECTION_SELECTABLE) != 0);
 
 	if (parent_layer_collection != NULL) {
-		is_visible &= (parent_layer_collection->flag_evaluated & COLLECTION_VISIBLE) != 0;
+		is_visible &= layer_collection_visible_get(eval_ctx, parent_layer_collection);
 		is_selectable &= (parent_layer_collection->flag_evaluated & COLLECTION_SELECTABLE) != 0;
 		layer_collection->flag_evaluated &= parent_layer_collection->flag_evaluated;
 	}