Groups and collection: initial integration

Since we are ditching layers from Blender (2.8) we need a replacement to
control groups visibility. This commit introduces collections as the building
blocks for groups, allowing users to control visibility as well as overrides
for groups.

Features
========

* Groups now have collections
    This way you can change the visibility of a collection inside a group, and add
    overrides which are part of the group and are prioritized over other overrides.

* Outliner
    Groups can inspect their collections, change visibility, and add/remove members.

    To change an override of a group collection, you need to select an instance of
    the group, and then you can choose "group" in the collection properties editor
    to edit this group active collection instead of the view layer one.

* Dupli groups overrides
    We can now have multiple instances of the same group with an original "override"
    and different overrides depending on the collection the instanced object is part
    of.

Technical
=========

* Layers
    We use the same api for groups and scene as much as possible.

Reviewers: sergey (depsgraph), mont29 (read/write and user count)
Differential Revision: https://developer.blender.org/D2892

3 files changed, 183 insertions, 87 deletions

diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 323586b66b2..a91c83c68ef 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -253,6 +253,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname);
 static void direct_link_modifiers(FileData *fd, ListBase *lb);
 static BHead *find_bhead_from_code_name(FileData *fd, const short idcode, const char *name);
 static BHead *find_bhead_from_idname(FileData *fd, const char *idname);
+static void expand_scene_collection(FileData *fd, Main *mainvar, SceneCollection *sc);
 static SceneCollection *get_scene_collection_active_or_create(struct Scene *scene, struct ViewLayer *view_layer, const short flag);
 
 /* this function ensures that reports are printed,
@@ -5832,6 +5833,28 @@ static void lib_link_scene_collection(FileData *fd, Library *lib, SceneCollectio
 	}
 }
 
+static void lib_link_view_layer(FileData *fd, Library *lib, ViewLayer *view_layer)
+{
+	/* tag scene layer to update for collection tree evaluation */
+	view_layer->flag |= VIEW_LAYER_ENGINE_DIRTY;
+
+	for (FreestyleModuleConfig *fmc = view_layer->freestyle_config.modules.first; fmc; fmc = fmc->next) {
+		fmc->script = newlibadr(fd, lib, fmc->script);
+	}
+
+	for (FreestyleLineSet *fls = view_layer->freestyle_config.linesets.first; fls; fls = fls->next) {
+		fls->linestyle = newlibadr_us(fd, lib, fls->linestyle);
+		fls->group = newlibadr_us(fd, lib, fls->group);
+	}
+
+	for (Base *base = view_layer->object_bases.first; base; base = base->next) {
+		/* we only bump the use count for the collection objects */
+		base->object = newlibadr(fd, lib, base->object);
+		base->flag |= BASE_DIRTY_ENGINE_SETTINGS;
+		base->collection_properties = NULL;
+	}
+}
+
 static void lib_link_scene(FileData *fd, Main *main)
 {
 #ifdef USE_SETSCENE_CHECK
@@ -5980,24 +6003,7 @@ static void lib_link_scene(FileData *fd, Main *main)
 			lib_link_scene_collection(fd, sce->id.lib, sce->collection);
 
 			for (ViewLayer *view_layer = sce->view_layers.first; view_layer; view_layer = view_layer->next) {
-				/* tag scene layer to update for collection tree evaluation */
-				view_layer->flag |= VIEW_LAYER_ENGINE_DIRTY;
-
-				for (FreestyleModuleConfig *fmc = view_layer->freestyle_config.modules.first; fmc; fmc = fmc->next) {
-					fmc->script = newlibadr(fd, sce->id.lib, fmc->script);
-				}
-
-				for (FreestyleLineSet *fls = view_layer->freestyle_config.linesets.first; fls; fls = fls->next) {
-					fls->linestyle = newlibadr_us(fd, sce->id.lib, fls->linestyle);
-					fls->group = newlibadr_us(fd, sce->id.lib, fls->group);
-				}
-
-				for (Base *base = view_layer->object_bases.first; base; base = base->next) {
-					/* we only bump the use count for the collection objects */
-					base->object = newlibadr(fd, sce->id.lib, base->object);
-					base->flag |= BASE_DIRTY_ENGINE_SETTINGS;
-					base->collection_properties = NULL;
-				}
+				lib_link_view_layer(fd, sce->id.lib, view_layer);
 			}
 
 #ifdef USE_SETSCENE_CHECK
@@ -6139,6 +6145,31 @@ static void direct_link_layer_collections(FileData *fd, ListBase *lb)
 	}
 }
 
+static void direct_link_view_layer(FileData *fd, ViewLayer *view_layer)
+{
+	view_layer->stats = NULL;
+	link_list(fd, &view_layer->object_bases);
+	view_layer->basact = newdataadr(fd, view_layer->basact);
+	direct_link_layer_collections(fd, &view_layer->layer_collections);
+
+	if (view_layer->properties != NULL) {
+		view_layer->properties = newdataadr(fd, view_layer->properties);
+		BLI_assert(view_layer->properties != NULL);
+		IDP_DirectLinkGroup_OrFree(&view_layer->properties, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
+		BKE_view_layer_engine_settings_validate_layer(view_layer);
+	}
+
+	view_layer->id_properties = newdataadr(fd, view_layer->id_properties);
+	IDP_DirectLinkGroup_OrFree(&view_layer->id_properties, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
+
+	link_list(fd, &(view_layer->freestyle_config.modules));
+	link_list(fd, &(view_layer->freestyle_config.linesets));
+
+	view_layer->properties_evaluated = NULL;
+
+	BLI_listbase_clear(&view_layer->drawdata);
+}
+
 /**
  * Workspaces store a render layer pointer which can only be read after scene is read.
  */
@@ -6416,27 +6447,7 @@ static void direct_link_scene(FileData *fd, Scene *sce, Main *bmain)
 	/* insert into global old-new map for reading without UI (link_global accesses it again) */
 	link_glob_list(fd, &sce->view_layers);
 	for (view_layer = sce->view_layers.first; view_layer; view_layer = view_layer->next) {
-		view_layer->stats = NULL;
-		link_list(fd, &view_layer->object_bases);
-		view_layer->basact = newdataadr(fd, view_layer->basact);
-		direct_link_layer_collections(fd, &view_layer->layer_collections);
-
-		if (view_layer->properties != NULL) {
-			view_layer->properties = newdataadr(fd, view_layer->properties);
-			BLI_assert(view_layer->properties != NULL);
-			IDP_DirectLinkGroup_OrFree(&view_layer->properties, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
-			BKE_view_layer_engine_settings_validate_layer(view_layer);
-		}
-
-		view_layer->id_properties = newdataadr(fd, view_layer->id_properties);
-		IDP_DirectLinkGroup_OrFree(&view_layer->id_properties, (fd->flags & FD_FLAGS_SWITCH_ENDIAN), fd);
-
-		link_list(fd, &(view_layer->freestyle_config.modules));
-		link_list(fd, &(view_layer->freestyle_config.linesets));
-
-		view_layer->properties_evaluated = NULL;
-
-		BLI_listbase_clear(&view_layer->drawdata);
+		direct_link_view_layer(fd, view_layer);
 	}
 
 	sce->collection_properties = newdataadr(fd, sce->collection_properties);
@@ -7781,30 +7792,61 @@ static void direct_link_group(FileData *fd, Group *group)
 	link_list(fd, &group->gobject);
 
 	group->preview = direct_link_preview_image(fd, group->preview);
+
+	/* This runs before the very first doversion. */
+	if (group->collection != NULL) {
+		group->collection = newdataadr(fd, group->collection);
+		direct_link_scene_collection(fd, group->collection);
+	}
+
+	if (group->view_layer != NULL) {
+		group->view_layer = newdataadr(fd, group->view_layer);
+		direct_link_view_layer(fd, group->view_layer);
+	}
 }
 
 static void lib_link_group(FileData *fd, Main *main)
 {
 	for (Group *group = main->group.first; group; group = group->id.next) {
 		if (group->id.tag & LIB_TAG_NEED_LINK) {
+			group->id.tag &= ~LIB_TAG_NEED_LINK;
 			IDP_LibLinkProperty(group->id.properties, fd);
-			
-			bool add_us = false;
-			
-			for (GroupObject *go = group->gobject.first; go; go = go->next) {
-				go->ob = newlibadr_real_us(fd, group->id.lib, go->ob);
-				if (go->ob) {
-					go->ob->flag |= OB_FROMGROUP;
-					/* if group has an object, it increments user... */
-					add_us = true;
+
+			if (group->view_layer == NULL) {
+				/* Old file, this is required for doversion. */
+				bool add_us = false;
+
+				GroupObject *go, *gon;
+				go = group->gobject.first;
+				while (go) {
+					gon = go->next;
+					go->ob = newlibadr_real_us(fd, group->id.lib, go->ob);
+					if (go->ob != NULL) {
+						go->ob->flag |= OB_FROMGROUP;
+						/* If group has an object, it increments user... */
+						add_us = true;
+					}
+					else {
+						/* Remove NULL objects. */
+						BLI_remlink(&group->gobject, go);
+						MEM_freeN(go);
+					}
+					go =  gon;
 				}
+
+				if (add_us) {
+					id_us_ensure_real(&group->id);
+				}
+				/* The rest of the read code is only for new files, skip it. */
+				continue;
 			}
-			if (add_us) {
+
+			lib_link_scene_collection(fd, group->id.lib, group->collection);
+			lib_link_view_layer(fd, group->id.lib, group->view_layer);
+
+			if (!BLI_listbase_is_empty(&group->view_layer->object_bases)) {
 				id_us_ensure_real(&group->id);
 			}
-			BKE_group_object_unlink(group, NULL);	/* removes NULL entries */
-
-			group->id.tag &= ~LIB_TAG_NEED_LINK;
 		}
 	}
 }
@@ -9407,6 +9449,11 @@ static void expand_group(FileData *fd, Main *mainvar, Group *group)
 	for (go = group->gobject.first; go; go = go->next) {
 		expand_doit(fd, mainvar, go->ob);
 	}
+
+	if (group->collection != NULL) {
+		expand_scene_collection(fd, mainvar, group->collection);
+	}
+
 }
 
 static void expand_key(FileData *fd, Main *mainvar, Key *key)
@@ -10233,7 +10280,7 @@ static void give_base_to_objects(
 					scene_collection = get_scene_collection_active_or_create(scene, view_layer, FILE_ACTIVE_COLLECTION);
 				}
 
-				BKE_collection_object_add(scene, scene_collection, ob);
+				BKE_collection_object_add(&scene->id, scene_collection, ob);
 				base = BKE_view_layer_base_find(view_layer, ob);
 				BKE_scene_object_base_flag_sync_from_base(base);
 
@@ -10275,7 +10322,7 @@ static void give_base_to_groups(
 			ob = BKE_object_add_only_object(mainvar, OB_EMPTY, group->id.name + 2);
 			ob->type = OB_EMPTY;
 
-			BKE_collection_object_add(scene, scene_collection, ob);
+			BKE_collection_object_add(&scene->id, scene_collection, ob);
 			base = BKE_view_layer_base_find(view_layer, ob);
 
 			if (base->flag & BASE_SELECTABLED) {
@@ -10369,7 +10416,7 @@ static SceneCollection *get_scene_collection_active_or_create(struct Scene *scen
 		lc = BKE_layer_collection_get_active_ensure(scene, view_layer);
 	}
 	else {
-		SceneCollection *sc = BKE_collection_add(scene, NULL, NULL);
+		SceneCollection *sc = BKE_collection_add(&scene->id, NULL, COLLECTION_TYPE_NONE, NULL);
 		lc = BKE_collection_link(view_layer, sc);
 	}
 
@@ -10388,7 +10435,7 @@ static void link_object_postprocess(ID *id, Scene *scene, ViewLayer *view_layer,
 		ob->mode = OB_MODE_OBJECT;
 
 		sc =  get_scene_collection_active_or_create(scene, view_layer, flag);
-		BKE_collection_object_add(scene, sc, ob);
+		BKE_collection_object_add(&scene->id, sc, ob);
 		base = BKE_view_layer_base_find(view_layer, ob);
 		BKE_scene_object_base_flag_sync_from_base(base);
 
diff --git a/source/blender/blenloader/intern/versioning_280.c b/source/blender/blenloader/intern/versioning_280.c
index 3b5e248f58b..8f2295781bf 100644
--- a/source/blender/blenloader/intern/versioning_280.c
+++ b/source/blender/blenloader/intern/versioning_280.c
@@ -39,6 +39,7 @@
 #include "DNA_object_types.h"
 #include "DNA_camera_types.h"
 #include "DNA_gpu_types.h"
+#include "DNA_group_types.h"
 #include "DNA_lamp_types.h"
 #include "DNA_layer_types.h"
 #include "DNA_material_types.h"
@@ -52,6 +53,7 @@
 #include "BKE_collection.h"
 #include "BKE_customdata.h"
 #include "BKE_freestyle.h"
+#include "BKE_group.h"
 #include "BKE_idprop.h"
 #include "BKE_layer.h"
 #include "BKE_main.h"
@@ -171,7 +173,7 @@ void do_versions_after_linking_280(Main *main)
 		for (Scene *scene = main->scene.first; scene; scene = scene->id.next) {
 			/* since we don't have access to FileData we check the (always valid) first render layer instead */
 			if (scene->view_layers.first == NULL) {
-				SceneCollection *sc_master = BKE_collection_master(scene);
+				SceneCollection *sc_master = BKE_collection_master(&scene->id);
 				BLI_strncpy(sc_master->name, "Master Collection", sizeof(sc_master->name));
 
 				struct DoVersionSceneCollections {
@@ -242,7 +244,7 @@ void do_versions_after_linking_280(Main *main)
 									             layer + 1,
 									             collections[DO_VERSION_COLLECTION_VISIBLE].suffix);
 									collections[DO_VERSION_COLLECTION_VISIBLE].collections[layer] =
-									        BKE_collection_add(scene, sc_master, name);
+									        BKE_collection_add(&scene->id, sc_master, COLLECTION_TYPE_NONE, name);
 									collections[DO_VERSION_COLLECTION_VISIBLE].created |= (1 << layer);
 								}
 
@@ -254,12 +256,16 @@ void do_versions_after_linking_280(Main *main)
 									             "Collection %d%s",
 									             layer + 1,
 									             collections[collection_index].suffix);
-									collections[collection_index].collections[layer] = BKE_collection_add(scene, sc_parent, name);
+									collections[collection_index].collections[layer] = BKE_collection_add(
+									                                                       &scene->id,
+									                                                       sc_parent,
+									                                                       COLLECTION_TYPE_NONE,
+									                                                       name);
 									collections[collection_index].created |= (1 << layer);
 								}
 							}
 
-							BKE_collection_object_add(scene, collections[collection_index].collections[layer], base->object);
+							BKE_collection_object_add(&scene->id, collections[collection_index].collections[layer], base->object);
 						}
 
 						if (base->flag & SELECT) {
@@ -542,6 +548,43 @@ void do_versions_after_linking_280(Main *main)
 			BLI_freelistN(&scene->r.layers);
 		}
 	}
+
+	{
+		/* Since we don't have access to FileData we check the (always valid) master collection of the group. */
+		for (Group *group = main->group.first; group; group = group->id.next) {
+			if (group->collection == NULL) {
+				BKE_group_init(group);
+				SceneCollection *sc = GROUP_MASTER_COLLECTION(group);
+				SceneCollection *sc_hidden = NULL;
+
+				for (GroupObject *go = group->gobject.first; go; go = go->next) {
+					if (go->ob->lay & group->layer){
+						BKE_collection_object_add(&group->id, sc, go->ob);
+					}
+					else {
+						if (sc_hidden == NULL) {
+							sc_hidden = BKE_collection_add(&group->id, sc, COLLECTION_TYPE_GROUP_INTERNAL, "Hidden");
+						}
+				        BKE_collection_object_add(&group->id, sc_hidden, go->ob);
+					}
+				}
+
+				if (sc_hidden != NULL) {
+					LayerCollection *layer_collection_master, *layer_collection_hidden;
+
+					layer_collection_master = group->view_layer->layer_collections.first;
+					layer_collection_hidden = layer_collection_master->layer_collections.first;
+
+					layer_collection_hidden->flag &= ~COLLECTION_VISIBLE;
+				}
+			}
+
+			GroupObject *go;
+			while ((go = BLI_pophead(&group->gobject))) {
+				MEM_freeN(go);
+			}
+		}
+	}
 }
 
 static void do_version_layer_collections_idproperties(ListBase *lb)
diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c
index 08ec2231e38..36ce277e4cf 100644
--- a/source/blender/blenloader/intern/writefile.c
+++ b/source/blender/blenloader/intern/writefile.c
@@ -168,6 +168,7 @@
 #include "BKE_curve.h"
 #include "BKE_constraint.h"
 #include "BKE_global.h" // for G
+#include "BKE_group.h"
 #include "BKE_idcode.h"
 #include "BKE_library.h" // for  set_listbasepointers
 #include "BKE_library_override.h"
@@ -1348,9 +1349,13 @@ static void write_particlesettings(WriteData *wd, ParticleSettings *part)
 			if (dw->ob != NULL) {
 				dw->index = 0;
 				if (part->dup_group) { /* can be NULL if lining fails or set to None */
-					for (GroupObject *go = part->dup_group->gobject.first;
-					     go && go->ob != dw->ob;
-					     go = go->next, dw->index++);
+					FOREACH_GROUP_OBJECT(part->dup_group, object)
+					{
+						if (object != dw->ob) {
+							dw->index++;
+						}
+					}
+					FOREACH_GROUP_OBJECT_END
 				}
 			}
 			writestruct(wd, DATA, ParticleDupliWeight, 1, dw);
@@ -2612,6 +2617,28 @@ static void write_layer_collections(WriteData *wd, ListBase *lb)
 	}
 }
 
+static void write_view_layer(WriteData *wd, ViewLayer *view_layer)
+{
+	writestruct(wd, DATA, ViewLayer, 1, view_layer);
+	writelist(wd, DATA, Base, &view_layer->object_bases);
+	if (view_layer->properties) {
+		IDP_WriteProperty(view_layer->properties, wd);
+	}
+
+	if (view_layer->id_properties) {
+		IDP_WriteProperty(view_layer->id_properties, wd);
+	}
+
+	for (FreestyleModuleConfig *fmc = view_layer->freestyle_config.modules.first; fmc; fmc = fmc->next) {
+		writestruct(wd, DATA, FreestyleModuleConfig, 1, fmc);
+	}
+
+	for (FreestyleLineSet *fls = view_layer->freestyle_config.linesets.first; fls; fls = fls->next) {
+		writestruct(wd, DATA, FreestyleLineSet, 1, fls);
+	}
+	write_layer_collections(wd, &view_layer->layer_collections);
+}
+
 static void write_scene(WriteData *wd, Scene *sce)
 {
 	/* write LibData */
@@ -2795,26 +2822,7 @@ static void write_scene(WriteData *wd, Scene *sce)
 	write_scene_collection(wd, sce->collection);
 
 	for (ViewLayer *view_layer = sce->view_layers.first; view_layer; view_layer = view_layer->next) {
-		writestruct(wd, DATA, ViewLayer, 1, view_layer);
-		writelist(wd, DATA, Base, &view_layer->object_bases);
-
-		if (view_layer->properties) {
-			IDP_WriteProperty(view_layer->properties, wd);
-		}
-
-		if (view_layer->id_properties) {
-			IDP_WriteProperty(view_layer->id_properties, wd);
-		}
-
-		for (FreestyleModuleConfig *fmc = view_layer->freestyle_config.modules.first; fmc; fmc = fmc->next) {
-			writestruct(wd, DATA, FreestyleModuleConfig, 1, fmc);
-		}
-
-		for (FreestyleLineSet *fls = view_layer->freestyle_config.linesets.first; fls; fls = fls->next) {
-			writestruct(wd, DATA, FreestyleLineSet, 1, fls);
-		}
-
-		write_layer_collections(wd, &view_layer->layer_collections);
+		write_view_layer(wd, view_layer);
 	}
 
 	if (sce->layer_properties) {
@@ -3233,10 +3241,8 @@ static void write_group(WriteData *wd, Group *group)
 		write_iddata(wd, &group->id);
 
 		write_previews(wd, group->preview);
-
-		for (GroupObject *go = group->gobject.first; go; go = go->next) {
-			writestruct(wd, DATA, GroupObject, 1, go);
-		}
+		write_scene_collection(wd, group->collection);
+		write_view_layer(wd, group->view_layer);
 	}
 }