Render Layers and Collections (merge from render-layers)

Design Documents
----------------

* https://wiki.blender.org/index.php/Dev:2.8/Source/Layers

* https://wiki.blender.org/index.php/Dev:2.8/Source/DataDesignRevised

User Commit Log
---------------

* New Layer and Collection system to replace render layers and viewport layers.

* A layer is a set of collections of objects (and their drawing options) required for specific tasks.

* A collection is a set of objects, equivalent of the old layers in Blender. A collection can be shared across multiple layers.

* All Scenes have a master collection that all other collections are children of.

* New collection "context" tab (in Properties Editor)

* New temporary viewport "collections" panel to control per-collection
visibility

Missing User Features
---------------------

* Collection "Filter"
  Option to add objects based on their names

* Collection Manager operators
  The existing buttons  are placeholders

* Collection Manager drawing
  The editor main region is empty

* Collection Override

* Per-Collection engine settings
  This will come as a separate commit, as part of the clay-engine branch

Dev Commit Log
--------------

* New DNA file (DNA_layer_types.h) with the new structs
  We are replacing Base by a new extended Base while keeping it backward
  compatible with some legacy settings (i.e., lay, flag_legacy).

  Renamed all Base to BaseLegacy to make it clear the areas of code that
  still need to be converted

  Note: manual changes were required on - deg_builder_nodes.h, rna_object.c, KX_Light.cpp

* Unittesting for main syncronization requirements
  - read, write, add/copy/remove objects, copy scene, collection
  link/unlinking, context)

* New Editor: Collection Manager
  Based on patch by Julian Eisel
  This is extracted from the layer-manager branch. With the following changes:

    - Renamed references of layer manager to collections manager

    - I doesn't include the editors/space_collections/ draw and util files

    - The drawing code itself will be implemented separately by Julian

* Base / Object:
  A little note about them. Original Blender code would try to keep them
  in sync through the code, juggling flags back and forth. This will now
  be handled by Depsgraph, keeping Object and Bases more separated
  throughout the non-rendering code.

  Scene.base is being cleared in doversion, and the old viewport drawing
  code was poorly converted to use the new bases while the new viewport
  code doesn't get merged and replace the old one.

Python API Changes
------------------

```
- scene.layers
+ # no longer exists

- scene.objects
+ scene.scene_layers.active.objects

- scene.objects.active
+ scene.render_layers.active.objects.active

- bpy.context.scene.objects.link()
+ bpy.context.scene_collection.objects.link()

- bpy_extras.object_utils.object_data_add(context, obdata, operator=None, use_active_layer=True, name=None)
+ bpy_extras.object_utils.object_data_add(context, obdata, operator=None, name=None)

- bpy.context.object.select
+ bpy.context.object.select = True
+ bpy.context.object.select = False
+ bpy.context.object.select_get()
+ bpy.context.object.select_set(action='SELECT')
+ bpy.context.object.select_set(action='DESELECT')

-AddObjectHelper.layers
+ # no longer exists
```

1 files changed, 116 insertions, 199 deletions

diff --git a/source/blender/editors/object/object_relations.c b/source/blender/editors/object/object_relations.c
index b5fbe4ba586..c79c702837a 100644
--- a/source/blender/editors/object/object_relations.c
+++ b/source/blender/editors/object/object_relations.c
@@ -63,6 +63,7 @@
 #include "BKE_animsys.h"
 #include "BKE_armature.h"
 #include "BKE_camera.h"
+#include "BKE_collection.h"
 #include "BKE_context.h"
 #include "BKE_constraint.h"
 #include "BKE_curve.h"
@@ -74,6 +75,7 @@
 #include "BKE_fcurve.h"
 #include "BKE_lamp.h"
 #include "BKE_lattice.h"
+#include "BKE_layer.h"
 #include "BKE_library.h"
 #include "BKE_library_query.h"
 #include "BKE_library_remap.h"
@@ -331,6 +333,7 @@ static int make_proxy_exec(bContext *C, wmOperator *op)
 	Object *ob, *gob = ED_object_active_context(C);
 	GroupObject *go;
 	Scene *scene = CTX_data_scene(C);
+	SceneLayer *sl = CTX_data_scene_layer(C);
 
 	if (gob->dup_group != NULL) {
 		go = BLI_findlink(&gob->dup_group->gobject, RNA_enum_get(op->ptr, "object"));
@@ -343,13 +346,13 @@ static int make_proxy_exec(bContext *C, wmOperator *op)
 
 	if (ob) {
 		Object *newob;
-		Base *newbase, *oldbase = BASACT;
+		BaseLegacy *newbase, *oldbase = BASACT;
 		char name[MAX_ID_NAME + 4];
 
 		BLI_snprintf(name, sizeof(name), "%s_proxy", ((ID *)(gob ? gob : ob))->name + 2);
 
 		/* Add new object for the proxy */
-		newob = BKE_object_add(bmain, scene, OB_EMPTY, name);
+		newob = BKE_object_add(bmain, scene, sl, OB_EMPTY, name);
 
 		/* set layers OK */
 		newbase = BASACT;    /* BKE_object_add sets active... */
@@ -1305,119 +1308,6 @@ void OBJECT_OT_track_set(wmOperatorType *ot)
 	ot->prop = RNA_def_enum(ot->srna, "type", prop_make_track_types, 0, "Type", "");
 }
 
-/************************** Move to Layer Operator *****************************/
-
-static unsigned int move_to_layer_init(bContext *C, wmOperator *op)
-{
-	int values[20], a;
-	unsigned int lay = 0;
-
-	if (!RNA_struct_property_is_set(op->ptr, "layers")) {
-		/* note: layers are set in bases, library objects work for this */
-		CTX_DATA_BEGIN (C, Base *, base, selected_bases)
-		{
-			lay |= base->lay;
-		}
-		CTX_DATA_END;
-
-		for (a = 0; a < 20; a++)
-			values[a] = (lay & (1 << a)) != 0;
-
-		RNA_boolean_set_array(op->ptr, "layers", values);
-	}
-	else {
-		RNA_boolean_get_array(op->ptr, "layers", values);
-
-		for (a = 0; a < 20; a++)
-			if (values[a])
-				lay |= (1 << a);
-	}
-
-	return lay;
-}
-
-static int move_to_layer_invoke(bContext *C, wmOperator *op, const wmEvent *event)
-{
-	View3D *v3d = CTX_wm_view3d(C);
-	if (v3d && v3d->localvd) {
-		return WM_operator_confirm_message(C, op, "Move out of Local View");
-	}
-	else {
-		move_to_layer_init(C, op);
-		return WM_operator_props_popup(C, op, event);
-	}
-}
-
-static int move_to_layer_exec(bContext *C, wmOperator *op)
-{
-	Main *bmain = CTX_data_main(C);
-	Scene *scene = CTX_data_scene(C);
-	View3D *v3d = CTX_wm_view3d(C);
-	unsigned int lay, local;
-	/* bool is_lamp = false; */ /* UNUSED */
-
-	lay = move_to_layer_init(C, op);
-	lay &= 0xFFFFFF;
-
-	if (lay == 0) return OPERATOR_CANCELLED;
-
-	if (v3d && v3d->localvd) {
-		/* now we can move out of localview. */
-		/* note: layers are set in bases, library objects work for this */
-		CTX_DATA_BEGIN (C, Base *, base, selected_bases)
-		{
-			lay = base->lay & ~v3d->lay;
-			base->lay = lay;
-			base->object->lay = lay;
-			base->object->flag &= ~SELECT;
-			base->flag &= ~SELECT;
-			/* if (base->object->type == OB_LAMP) is_lamp = true; */
-		}
-		CTX_DATA_END;
-	}
-	else {
-		/* normal non localview operation */
-		/* note: layers are set in bases, library objects work for this */
-		CTX_DATA_BEGIN (C, Base *, base, selected_bases)
-		{
-			/* upper byte is used for local view */
-			local = base->lay & 0xFF000000;
-			base->lay = lay + local;
-			base->object->lay = base->lay;
-			/* if (base->object->type == OB_LAMP) is_lamp = true; */
-		}
-		CTX_DATA_END;
-	}
-
-	/* warning, active object may be hidden now */
-
-	WM_event_add_notifier(C, NC_OBJECT | ND_DRAW, scene);
-	WM_event_add_notifier(C, NC_SCENE | ND_LAYER_CONTENT, scene);
-
-	DAG_relations_tag_update(bmain);
-
-	return OPERATOR_FINISHED;
-}
-
-void OBJECT_OT_move_to_layer(wmOperatorType *ot)
-{
-	/* identifiers */
-	ot->name = "Move to Layer";
-	ot->description = "Move the object to different layers";
-	ot->idname = "OBJECT_OT_move_to_layer";
-
-	/* api callbacks */
-	ot->invoke = move_to_layer_invoke;
-	ot->exec = move_to_layer_exec;
-	ot->poll = ED_operator_objectmode;
-
-	/* flags */
-	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
-
-	/* properties */
-	RNA_def_boolean_layer_member(ot->srna, "layers", 20, NULL, "Layer", "");
-}
-
 /************************** Link to Scene Operator *****************************/
 
 #if 0
@@ -1440,20 +1330,6 @@ static void link_to_scene(Main *UNUSED(bmain), unsigned short UNUSED(nr))
 }
 #endif
 
-Base *ED_object_scene_link(Scene *scene, Object *ob)
-{
-	Base *base;
-
-	if (BKE_scene_base_find(scene, ob)) {
-		return NULL;
-	}
-
-	base = BKE_scene_base_add(scene, ob);
-	id_us_plus(&ob->id);
-
-	return base;
-}
-
 static int make_links_scene_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene_to = BLI_findlink(&CTX_data_main(C)->scene, RNA_enum_get(op->ptr, "scene"));
@@ -1473,9 +1349,10 @@ static int make_links_scene_exec(bContext *C, wmOperator *op)
 		return OPERATOR_CANCELLED;
 	}
 
+	SceneCollection *sc_to = BKE_collection_master(scene_to);
 	CTX_DATA_BEGIN (C, Base *, base, selected_bases)
 	{
-		ED_object_scene_link(scene_to, base->object);
+		BKE_collection_object_add(scene_to, sc_to, base->object);
 	}
 	CTX_DATA_END;
 
@@ -1531,7 +1408,6 @@ static bool allow_make_links_data(const int type, Object *ob_src, Object *ob_dst
 static int make_links_data_exec(bContext *C, wmOperator *op)
 {
 	Main *bmain = CTX_data_main(C);
-	Scene *scene = CTX_data_scene(C);
 	const int type = RNA_enum_get(op->ptr, "type");
 	Object *ob_src;
 	ID *obdata_id;
@@ -1594,12 +1470,12 @@ static int make_links_data_exec(bContext *C, wmOperator *op)
 						LinkNode *group_node;
 
 						/* first clear groups */
-						BKE_object_groups_clear(scene, base_dst, ob_dst);
+						BKE_object_groups_clear(ob_dst);
 
 						/* now add in the groups from the link nodes */
 						for (group_node = ob_groups; group_node; group_node = group_node->next) {
 							if (ob_dst->dup_group != group_node->link) {
-								BKE_group_object_add(group_node->link, ob_dst, scene, base_dst);
+								BKE_group_object_add(group_node->link, ob_dst);
 							}
 							else {
 								is_cycle = true;
@@ -1732,48 +1608,82 @@ void OBJECT_OT_make_links_data(wmOperatorType *ot)
 
 /**************************** Make Single User ********************************/
 
+static Object *single_object_users_object(Main *bmain, Scene *scene, Object *ob, const bool copy_groups)
+{
+	if (!ID_IS_LINKED_DATABLOCK(ob) && ob->id.us > 1) {
+		/* base gets copy of object */
+		Object *obn = ID_NEW_SET(ob, BKE_object_copy(bmain, ob));
+
+		if (copy_groups) {
+			if (ob->flag & OB_FROMGROUP) {
+				obn->flag |= OB_FROMGROUP;
+			}
+		}
+		else {
+			/* copy already clears */
+		}
+		/* remap gpencil parenting */
+
+		if (scene->gpd) {
+			bGPdata *gpd = scene->gpd;
+			for (bGPDlayer *gpl = gpd->layers.first; gpl; gpl = gpl->next) {
+				if (gpl->parent == ob) {
+					gpl->parent = obn;
+				}
+			}
+		}
+
+		id_us_min(&ob->id);
+		return obn;
+	}
+	return NULL;
+}
+
+static void libblock_relink_scene_collection(SceneCollection *sc)
+{
+	for (LinkData *link = sc->objects.first; link; link = link->next) {
+		BKE_libblock_relink_to_newid(link->data);
+	}
+
+	for (SceneCollection *nsc = sc->scene_collections.first; nsc; nsc = nsc->next) {
+		libblock_relink_scene_collection(nsc);
+	}
+}
+
+static void single_object_users_scene_collection(Main *bmain, Scene *scene, SceneCollection *sc, const int flag, const bool copy_groups)
+{
+	for (LinkData *link = sc->objects.first; link; link = link->next) {
+		Object *ob = link->data;
+		/* an object may be in more than one collection */
+		if ((ob->id.newid == NULL) && ((ob->flag & flag) == flag)) {
+			link->data = single_object_users_object(bmain, scene, link->data, copy_groups);
+		}
+	}
+
+	/* we reset filter objects because they should be regenerated after this */
+	BLI_freelistN(&sc->filter_objects);
+
+	for (SceneCollection *nsc = sc->scene_collections.first; nsc; nsc = nsc->next) {
+		single_object_users_scene_collection(bmain, scene, nsc, flag, copy_groups);
+	}
+}
+
 /* Warning, sets ID->newid pointers of objects and groups, but does not clear them. */
 static void single_object_users(Main *bmain, Scene *scene, View3D *v3d, const int flag, const bool copy_groups)
 {
-	Base *base;
-	Object *ob, *obn;
 	Group *group, *groupn;
 	GroupObject *go;
 
 	clear_sca_new_poins();  /* BGE logic */
 
-	/* duplicate (must set newid) */
-	for (base = FIRSTBASE; base; base = base->next) {
-		ob = base->object;
-
-		if ((base->flag & flag) == flag) {
-			if (!ID_IS_LINKED_DATABLOCK(ob) && ob->id.us > 1) {
-				/* base gets copy of object */
-				base->object = obn = ID_NEW_SET(ob, BKE_object_copy(bmain, ob));
-
-				if (copy_groups) {
-					if (ob->flag & OB_FROMGROUP) {
-						obn->flag |= OB_FROMGROUP;
-					}
-				}
-				else {
-					/* copy already clears */
-				}
-				/* remap gpencil parenting */
-
-				if (scene->gpd) {
-					bGPdata *gpd = scene->gpd;
-					for (bGPDlayer *gpl = gpd->layers.first; gpl; gpl = gpl->next) {
-						if (gpl->parent == ob) {
-							gpl->parent = obn;
-						}
-					}
-				}
-
-				base->flag = obn->flag;
+	/* duplicate all the objects of the scene */
+	SceneCollection *msc = BKE_collection_master(scene);
+	single_object_users_scene_collection(bmain, scene, msc, flag, copy_groups);
 
-				id_us_min(&ob->id);
-			}
+	/* loop over SceneLayers and assign the pointers accordingly */
+	for (SceneLayer *sl = scene->render_layers.first; sl; sl = sl->next) {
+		for (Base *base = sl->object_bases.first; base; base = base->next) {
+			ID_NEW_REMAP(base->object);
 		}
 	}
 
@@ -1806,27 +1716,29 @@ static void single_object_users(Main *bmain, Scene *scene, View3D *v3d, const in
 	if (v3d) ID_NEW_REMAP(v3d->camera);
 
 	/* object and group pointers */
-	for (base = FIRSTBASE; base; base = base->next) {
-		BKE_libblock_relink_to_newid(&base->object->id);
-	}
+	libblock_relink_scene_collection(msc);
 
 	set_sca_new_poins();
+
+	/* TODO redo filter */
+	TODO_LAYER_SYNC_FILTER
 }
 
 /* not an especially efficient function, only added so the single user
  * button can be functional.*/
 void ED_object_single_user(Main *bmain, Scene *scene, Object *ob)
 {
-	Base *base;
-	const bool copy_groups = false;
-
-	for (base = FIRSTBASE; base; base = base->next) {
-		if (base->object == ob) base->flag |=  OB_DONE;
-		else base->flag &= ~OB_DONE;
+	Object *ob_iter;
+	FOREACH_SCENE_OBJECT(scene, ob_iter)
+	{
+		ob_iter->flag &= ~OB_DONE;
 	}
+	FOREACH_SCENE_OBJECT_END
 
-	single_object_users(bmain, scene, NULL, OB_DONE, copy_groups);
+	/* tag only the one object */
+	ob->flag |= OB_DONE;
 
+	single_object_users(bmain, scene, NULL, OB_DONE, false);
 	BKE_main_id_clear_newpoins(bmain);
 }
 
@@ -1857,15 +1769,17 @@ static void single_obdata_users(Main *bmain, Scene *scene, const int flag)
 	Lamp *la;
 	Curve *cu;
 	/* Camera *cam; */
-	Base *base;
+	BaseLegacy *base;
 	Mesh *me;
 	Lattice *lat;
 	ID *id;
 	int a;
 
+	TODO_LAYER; /* need to use scene->collection base instead of scene->bases */
+
 	for (base = FIRSTBASE; base; base = base->next) {
 		ob = base->object;
-		if (!ID_IS_LINKED_DATABLOCK(ob) && (base->flag & flag) == flag) {
+		if (!ID_IS_LINKED_DATABLOCK(ob) && (base->flag_legacy & flag) == flag) {
 			id = ob->data;
 
 			if (id && id->us > 1 && !ID_IS_LINKED_DATABLOCK(id)) {
@@ -1939,31 +1853,27 @@ static void single_obdata_users(Main *bmain, Scene *scene, const int flag)
 	}
 }
 
-static void single_object_action_users(Scene *scene, const int flag)
+static void single_object_action_users(Scene *scene, SceneLayer *sl, const int flag)
 {
 	Object *ob;
-	Base *base;
 
-	for (base = FIRSTBASE; base; base = base->next) {
-		ob = base->object;
-		if (!ID_IS_LINKED_DATABLOCK(ob) && (flag == 0 || (base->flag & SELECT)) ) {
+	FOREACH_OBJECT_FLAG(scene, sl, flag, ob)
+	    if (!ID_IS_LINKED_DATABLOCK(ob)) {
 			DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
 			BKE_animdata_copy_id_action(&ob->id, false);
 		}
-	}
+	FOREACH_OBJECT_FLAG_END
 }
 
-static void single_mat_users(Main *bmain, Scene *scene, const int flag, const bool do_textures)
+static void single_mat_users(Main *bmain, Scene *scene, SceneLayer *sl, const int flag, const bool do_textures)
 {
-	Object *ob;
-	Base *base;
 	Material *ma, *man;
 	Tex *tex;
 	int a, b;
 
-	for (base = FIRSTBASE; base; base = base->next) {
-		ob = base->object;
-		if (!ID_IS_LINKED_DATABLOCK(ob) && (flag == 0 || (base->flag & SELECT)) ) {
+	Object *ob;
+	FOREACH_OBJECT_FLAG(scene, sl, flag, ob)
+	    if (!ID_IS_LINKED_DATABLOCK(ob)) {
 			for (a = 1; a <= ob->totcol; a++) {
 				ma = give_current_material(ob, a);
 				if (ma) {
@@ -1992,7 +1902,7 @@ static void single_mat_users(Main *bmain, Scene *scene, const int flag, const bo
 				}
 			}
 		}
-	}
+	FOREACH_OBJECT_FLAG_END
 }
 
 static void do_single_tex_user(Main *bmain, Tex **from)
@@ -2096,7 +2006,7 @@ void ED_object_single_users(Main *bmain, Scene *scene, const bool full, const bo
 
 	if (full) {
 		single_obdata_users(bmain, scene, 0);
-		single_object_action_users(scene, 0);
+		single_object_action_users(scene, NULL, 0);
 		single_mat_users_expand(bmain);
 		single_tex_users_expand(bmain);
 	}
@@ -2194,13 +2104,13 @@ static bool make_local_all__instance_indirect_unused(Main *bmain, Scene *scene)
 
 	for (ob = bmain->object.first; ob; ob = ob->id.next) {
 		if (ID_IS_LINKED_DATABLOCK(ob) && (ob->id.us == 0)) {
-			Base *base;
+			BaseLegacy *base;
 
 			id_us_plus(&ob->id);
 
 			base = BKE_scene_base_add(scene, ob);
-			base->flag |= SELECT;
-			base->object->flag = base->flag;
+			base->flag_legacy |= SELECT;
+			BKE_scene_base_flag_sync_from_base(base);
 			DAG_id_tag_update(&ob->id, OB_RECALC_OB | OB_RECALC_DATA | OB_RECALC_TIME);
 
 			changed = true;
@@ -2366,13 +2276,20 @@ static int make_single_user_exec(bContext *C, wmOperator *op)
 {
 	Main *bmain = CTX_data_main(C);
 	Scene *scene = CTX_data_scene(C);
+	SceneLayer *sl = CTX_data_scene_layer(C);
 	View3D *v3d = CTX_wm_view3d(C); /* ok if this is NULL */
 	const int flag = (RNA_enum_get(op->ptr, "type") == MAKE_SINGLE_USER_SELECTED) ? SELECT : 0;
 	const bool copy_groups = false;
 	bool update_deps = false;
 
 	if (RNA_boolean_get(op->ptr, "object")) {
-		single_object_users(bmain, scene, v3d, flag, copy_groups);
+		if (flag == SELECT) {
+			BKE_scene_layer_selected_objects_tag(sl, OB_DONE);
+			single_object_users(bmain, scene, v3d, OB_DONE, copy_groups);
+		}
+		else {
+			single_object_users(bmain, scene, v3d, 0, copy_groups);
+		}
 
 		/* needed since object relationships may have changed */
 		update_deps = true;
@@ -2383,7 +2300,7 @@ static int make_single_user_exec(bContext *C, wmOperator *op)
 	}
 
 	if (RNA_boolean_get(op->ptr, "material")) {
-		single_mat_users(bmain, scene, flag, RNA_boolean_get(op->ptr, "texture"));
+		single_mat_users(bmain, scene, sl, flag, RNA_boolean_get(op->ptr, "texture"));
 	}
 
 #if 0 /* can't do this separate from materials */
@@ -2391,7 +2308,7 @@ static int make_single_user_exec(bContext *C, wmOperator *op)
 		single_mat_users(scene, flag, true);
 #endif
 	if (RNA_boolean_get(op->ptr, "animation")) {
-		single_object_action_users(scene, flag);
+		single_object_action_users(scene, sl, flag);
 	}
 
 	BKE_main_id_clear_newpoins(bmain);
@@ -2438,7 +2355,7 @@ void OBJECT_OT_make_single_user(wmOperatorType *ot)
 
 static int drop_named_material_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
-	Base *base = ED_view3d_give_base_under_cursor(C, event->mval);
+	BaseLegacy *base = ED_view3d_give_base_under_cursor(C, event->mval);
 	Material *ma;
 	char name[MAX_ID_NAME - 2];