Collections: preserve exclude flag of child collections when unexcluding

Excluding a collection also changes the exclude setting on all child collections
so that it is possible to selectively enable some children without the parent
being enabled.

This change makes it so that if you unexclude the parent, the exclude setting
of children are restored again instead of being permanently lost.

Original patch by Szymon with modifications by Brecht.

Differential Revision: https://developer.blender.org/D7016

5 files changed, 48 insertions, 39 deletions

diff --git a/source/blender/blenkernel/BKE_layer.h b/source/blender/blenkernel/BKE_layer.h
index 7059675ec7d..7a0252e6813 100644
--- a/source/blender/blenkernel/BKE_layer.h
+++ b/source/blender/blenkernel/BKE_layer.h
@@ -137,6 +137,7 @@ void BKE_layer_collection_set_visible(struct ViewLayer *view_layer,
                                       struct LayerCollection *lc,
                                       const bool visible,
                                       const bool hierarchy);
+void BKE_layer_collection_set_flag(struct LayerCollection *lc, const int flag, const bool value);
 
 /* evaluation */
 
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index d8f0bda8c22..f6e80d66ad1 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -1359,6 +1359,49 @@ void BKE_layer_collection_set_visible(ViewLayer *view_layer,
   }
 }
 
+/**
+ * Set layer collection hide/exclude/indirect flag on a layer collection.
+ * recursively.
+ */
+static void layer_collection_flag_recursive_set(LayerCollection *lc,
+                                                const int flag,
+                                                const bool value,
+                                                const bool restore_flag)
+{
+  if (flag == LAYER_COLLECTION_EXCLUDE) {
+    /* For exclude flag, we remember the state the children had before
+     * excluding and restoring it when enabling the parent collection again. */
+    if (value) {
+      if (restore_flag) {
+        SET_FLAG_FROM_TEST(
+            lc->flag, (lc->flag & LAYER_COLLECTION_EXCLUDE), LAYER_COLLECTION_PREVIOUSLY_EXCLUDED);
+      }
+      else {
+        lc->flag &= ~LAYER_COLLECTION_PREVIOUSLY_EXCLUDED;
+      }
+
+      lc->flag |= flag;
+    }
+    else {
+      if (!(lc->flag & LAYER_COLLECTION_PREVIOUSLY_EXCLUDED)) {
+        lc->flag &= ~flag;
+      }
+    }
+  }
+  else {
+    SET_FLAG_FROM_TEST(lc->flag, value, flag);
+  }
+
+  LISTBASE_FOREACH (LayerCollection *, nlc, &lc->layer_collections) {
+    layer_collection_flag_recursive_set(nlc, flag, value, true);
+  }
+}
+
+void BKE_layer_collection_set_flag(LayerCollection *lc, const int flag, const bool value)
+{
+  layer_collection_flag_recursive_set(lc, flag, value, false);
+}
+
 /* ---------------------------------------------------------------------- */
 
 static LayerCollection *find_layer_collection_by_scene_collection(LayerCollection *lc,
diff --git a/source/blender/editors/space_outliner/outliner_collections.c b/source/blender/editors/space_outliner/outliner_collections.c
index 5a0ed954909..4b6241d45ee 100644
--- a/source/blender/editors/space_outliner/outliner_collections.c
+++ b/source/blender/editors/space_outliner/outliner_collections.c
@@ -845,20 +845,6 @@ static bool collections_indirect_only_clear_poll(bContext *C)
   return collections_view_layer_poll(C, true, LAYER_COLLECTION_INDIRECT_ONLY);
 }
 
-static void layer_collection_flag_recursive_set(LayerCollection *lc, int flag)
-{
-  LISTBASE_FOREACH (LayerCollection *, nlc, &lc->layer_collections) {
-    if (lc->flag & flag) {
-      nlc->flag |= flag;
-    }
-    else {
-      nlc->flag &= ~flag;
-    }
-
-    layer_collection_flag_recursive_set(nlc, flag);
-  }
-}
-
 static int collection_view_layer_exec(bContext *C, wmOperator *op)
 {
   Main *bmain = CTX_data_main(C);
@@ -883,15 +869,7 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
   GSetIterator collections_to_edit_iter;
   GSET_ITER (collections_to_edit_iter, data.collections_to_edit) {
     LayerCollection *lc = BLI_gsetIterator_getKey(&collections_to_edit_iter);
-
-    if (clear) {
-      lc->flag &= ~flag;
-    }
-    else {
-      lc->flag |= flag;
-    }
-
-    layer_collection_flag_recursive_set(lc, flag);
+    BKE_layer_collection_set_flag(lc, flag, !clear);
   }
 
   BLI_gset_free(data.collections_to_edit, NULL);
@@ -1468,8 +1446,7 @@ static int outliner_unhide_all_exec(bContext *C, wmOperator *UNUSED(op))
   /* Unhide all the collections. */
   LayerCollection *lc_master = view_layer->layer_collections.first;
   LISTBASE_FOREACH (LayerCollection *, lc_iter, &lc_master->layer_collections) {
-    lc_iter->flag &= ~LAYER_COLLECTION_HIDE;
-    layer_collection_flag_recursive_set(lc_iter, LAYER_COLLECTION_HIDE);
+    BKE_layer_collection_set_flag(lc_iter, LAYER_COLLECTION_HIDE, false);
   }
 
   /* Unhide all objects. */
diff --git a/source/blender/makesdna/DNA_layer_types.h b/source/blender/makesdna/DNA_layer_types.h
index a9293d18d41..4676e7fa313 100644
--- a/source/blender/makesdna/DNA_layer_types.h
+++ b/source/blender/makesdna/DNA_layer_types.h
@@ -173,6 +173,7 @@ enum {
   LAYER_COLLECTION_HOLDOUT = (1 << 5),
   LAYER_COLLECTION_INDIRECT_ONLY = (1 << 6),
   LAYER_COLLECTION_HIDE = (1 << 7),
+  LAYER_COLLECTION_PREVIOUSLY_EXCLUDED = (1 << 8),
 };
 
 /* Layer Collection->runtime_flag
diff --git a/source/blender/makesrna/intern/rna_layer.c b/source/blender/makesrna/intern/rna_layer.c
index 46a32256114..b99457056fe 100644
--- a/source/blender/makesrna/intern/rna_layer.c
+++ b/source/blender/makesrna/intern/rna_layer.c
@@ -299,19 +299,6 @@ static void rna_LayerCollection_hide_viewport_set(PointerRNA *ptr, bool value)
   rna_LayerCollection_flag_set(ptr, value, LAYER_COLLECTION_HIDE);
 }
 
-static void rna_LayerCollection_exclude_update_recursive(ListBase *lb, const bool exclude)
-{
-  LISTBASE_FOREACH (LayerCollection *, lc, lb) {
-    if (exclude) {
-      lc->flag |= LAYER_COLLECTION_EXCLUDE;
-    }
-    else {
-      lc->flag &= ~LAYER_COLLECTION_EXCLUDE;
-    }
-    rna_LayerCollection_exclude_update_recursive(&lc->layer_collections, exclude);
-  }
-}
-
 static void rna_LayerCollection_exclude_update(Main *bmain, Scene *UNUSED(scene), PointerRNA *ptr)
 {
   Scene *scene = (Scene *)ptr->owner_id;
@@ -320,7 +307,7 @@ static void rna_LayerCollection_exclude_update(Main *bmain, Scene *UNUSED(scene)
 
   /* Set/Unset it recursively to match the behavior of excluding via the menu or shortcuts. */
   const bool exclude = (lc->flag & LAYER_COLLECTION_EXCLUDE) != 0;
-  rna_LayerCollection_exclude_update_recursive(&lc->layer_collections, exclude);
+  BKE_layer_collection_set_flag(lc, LAYER_COLLECTION_EXCLUDE, exclude);
 
   BKE_layer_collection_sync(scene, view_layer);