Render Layers and Collections (merge from render-layers)

Design Documents
----------------

* https://wiki.blender.org/index.php/Dev:2.8/Source/Layers

* https://wiki.blender.org/index.php/Dev:2.8/Source/DataDesignRevised

User Commit Log
---------------

* New Layer and Collection system to replace render layers and viewport layers.

* A layer is a set of collections of objects (and their drawing options) required for specific tasks.

* A collection is a set of objects, equivalent of the old layers in Blender. A collection can be shared across multiple layers.

* All Scenes have a master collection that all other collections are children of.

* New collection "context" tab (in Properties Editor)

* New temporary viewport "collections" panel to control per-collection
visibility

Missing User Features
---------------------

* Collection "Filter"
  Option to add objects based on their names

* Collection Manager operators
  The existing buttons  are placeholders

* Collection Manager drawing
  The editor main region is empty

* Collection Override

* Per-Collection engine settings
  This will come as a separate commit, as part of the clay-engine branch

Dev Commit Log
--------------

* New DNA file (DNA_layer_types.h) with the new structs
  We are replacing Base by a new extended Base while keeping it backward
  compatible with some legacy settings (i.e., lay, flag_legacy).

  Renamed all Base to BaseLegacy to make it clear the areas of code that
  still need to be converted

  Note: manual changes were required on - deg_builder_nodes.h, rna_object.c, KX_Light.cpp

* Unittesting for main syncronization requirements
  - read, write, add/copy/remove objects, copy scene, collection
  link/unlinking, context)

* New Editor: Collection Manager
  Based on patch by Julian Eisel
  This is extracted from the layer-manager branch. With the following changes:

    - Renamed references of layer manager to collections manager

    - I doesn't include the editors/space_collections/ draw and util files

    - The drawing code itself will be implemented separately by Julian

* Base / Object:
  A little note about them. Original Blender code would try to keep them
  in sync through the code, juggling flags back and forth. This will now
  be handled by Depsgraph, keeping Object and Bases more separated
  throughout the non-rendering code.

  Scene.base is being cleared in doversion, and the old viewport drawing
  code was poorly converted to use the new bases while the new viewport
  code doesn't get merged and replace the old one.

Python API Changes
------------------

```
- scene.layers
+ # no longer exists

- scene.objects
+ scene.scene_layers.active.objects

- scene.objects.active
+ scene.render_layers.active.objects.active

- bpy.context.scene.objects.link()
+ bpy.context.scene_collection.objects.link()

- bpy_extras.object_utils.object_data_add(context, obdata, operator=None, use_active_layer=True, name=None)
+ bpy_extras.object_utils.object_data_add(context, obdata, operator=None, name=None)

- bpy.context.object.select
+ bpy.context.object.select = True
+ bpy.context.object.select = False
+ bpy.context.object.select_get()
+ bpy.context.object.select_set(action='SELECT')
+ bpy.context.object.select_set(action='DESELECT')

-AddObjectHelper.layers
+ # no longer exists
```

6 files changed, 155 insertions, 18 deletions

diff --git a/source/blender/makesdna/DNA_layer_types.h b/source/blender/makesdna/DNA_layer_types.h
new file mode 100644
index 00000000000..6f6c6d7d145
--- /dev/null
+++ b/source/blender/makesdna/DNA_layer_types.h
@@ -0,0 +1,112 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Contributor(s): Dalai Felinto
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+/** \file DNA_layer_types.h
+ *  \ingroup DNA
+ */
+
+#ifndef __DNA_LAYER_TYPES_H__
+#define __DNA_LAYER_TYPES_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "DNA_listBase.h"
+
+typedef struct Base {
+	struct Base *next, *prev;
+	short flag;
+	short refcount;
+	short sx, sy;
+	struct Object *object;
+	unsigned int selcol;
+	unsigned int lay;
+	int flag_legacy;
+	int pad;
+} Base;
+
+typedef struct CollectionOverride {
+	struct CollectionOverride *next, *prev;
+	char name[64]; /* MAX_NAME */
+	/* TODO proper data */
+} CollectionOverride;
+
+typedef struct LayerCollection {
+	struct LayerCollection *next, *prev;
+	struct SceneCollection *scene_collection;
+	short flag;
+	short pad[3];
+	ListBase object_bases;      /* (ObjectBase *)LinkData->data - synced with collection->objects and collection->filter_objects */
+	ListBase overrides;
+	ListBase layer_collections; /* synced with collection->collections */
+} LayerCollection;
+
+typedef struct SceneLayer {
+	struct SceneLayer *next, *prev;
+	char name[64]; /* MAX_NAME */
+	char engine[32]; /* render engine */
+	short active_collection;
+	short flag;
+	short pad[2];
+	ListBase object_bases;      /* ObjectBase */
+	struct Base *basact;
+	ListBase layer_collections; /* LayerCollection */
+} SceneLayer;
+
+typedef struct SceneCollection {
+	struct SceneCollection *next, *prev;
+	char name[64]; /* MAX_NAME */
+	char filter[64]; /* MAX_NAME */
+	int active_object_index; /* for UI */
+	int pad;
+	ListBase objects;           /* (Object *)LinkData->data */
+	ListBase filter_objects;    /* (Object *)LinkData->data */
+	ListBase scene_collections; /* nested collections */
+} SceneCollection;
+
+/* Base->flag */
+enum {
+	BASE_SELECTED         = (1 << 0),
+	BASE_VISIBLED         = (1 << 1),
+	BASE_SELECTABLED      = (1 << 2),
+	BASE_FROMDUPLI        = (1 << 3),
+};
+
+/* LayerCollection->flag */
+enum {
+	COLLECTION_VISIBLE    = (1 << 0),
+	COLLECTION_SELECTABLE = (1 << 1),
+	COLLECTION_FOLDED     = (1 << 2),
+};
+
+/* SceneLayer->flag */
+enum {
+	SCENE_LAYER_RENDER = (1 << 0),
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  /* __DNA_LAYER_TYPES_H__ */
+
diff --git a/source/blender/makesdna/DNA_object_types.h b/source/blender/makesdna/DNA_object_types.h
index d24c7faa9f5..60aaa4ace20 100644
--- a/source/blender/makesdna/DNA_object_types.h
+++ b/source/blender/makesdna/DNA_object_types.h
@@ -223,7 +223,10 @@ typedef struct Object {
 	float jump_speed;
 	float fall_speed;
 	unsigned char max_jumps;
-	char pad2[3];
+	char pad2;
+
+	/* Depsgraph */
+	short base_flag; /* used by depsgraph, flushed from base */
 
 	/** Collision mask settings */
 	unsigned short col_group, col_mask;
diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index 8ee15ef21a3..ba5e50ec441 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -47,6 +47,7 @@ extern "C" {
 #include "DNA_ID.h"
 #include "DNA_freestyle_types.h"
 #include "DNA_gpu_types.h"
+#include "DNA_layer_types.h"
 #include "DNA_userdef_types.h"
 
 struct CurveMapping;
@@ -65,18 +66,13 @@ struct bGPdata;
 struct bGPDbrush;
 struct MovieClip;
 struct ColorSpace;
+struct SceneCollection;
 
 /* ************************************************************* */
 /* Scene Data */
 
 /* Base - Wrapper for referencing Objects in a Scene */
-typedef struct Base {
-	struct Base *next, *prev;
-	unsigned int lay, selcol;
-	int flag;
-	short sx, sy;
-	struct Object *object;
-} Base;
+#define BaseLegacy Base
 
 /* ************************************************************* */
 /* Output Format Data */
@@ -1628,7 +1624,7 @@ typedef struct Scene {
 	struct Scene *set;
 	
 	ListBase base;
-	struct Base *basact;		/* active base */
+	struct BaseLegacy *basact;		/* active base */
 	struct Object *obedit;		/* name replaces old G.obedit */
 	
 	float cursor[3];			/* 3d cursor location */
@@ -1706,6 +1702,11 @@ typedef struct Scene {
 	struct RigidBodyWorld *rigidbody_world;
 
 	struct PreviewImage *preview;
+
+	ListBase render_layers;
+	struct SceneCollection *collection;
+	int active_layer;
+	int pad4;
 } Scene;
 
 /* **************** RENDERDATA ********************* */
@@ -1912,16 +1913,16 @@ extern const char *RE_engine_id_CYCLES;
 
 /* depricate this! */
 #define TESTBASE(v3d, base)  (                                                \
-	((base)->flag & SELECT) &&                                                \
+	((base)->flag_legacy & SELECT) &&                                         \
 	((base)->lay & v3d->lay) &&                                               \
 	(((base)->object->restrictflag & OB_RESTRICT_VIEW) == 0))
 #define TESTBASELIB(v3d, base)  (                                             \
-	((base)->flag & SELECT) &&                                                \
+	((base)->flag_legacy & SELECT) &&                                         \
 	((base)->lay & v3d->lay) &&                                               \
 	((base)->object->id.lib == NULL) &&                                       \
 	(((base)->object->restrictflag & OB_RESTRICT_VIEW) == 0))
 #define TESTBASELIB_BGMODE(v3d, scene, base)  (                               \
-	((base)->flag & SELECT) &&                                                \
+	((base)->flag_legacy & SELECT) &&                                         \
 	((base)->lay & (v3d ? v3d->lay : scene->lay)) &&                          \
 	((base)->object->id.lib == NULL) &&                                       \
 	(((base)->object->restrictflag & OB_RESTRICT_VIEW) == 0))
@@ -1930,7 +1931,7 @@ extern const char *RE_engine_id_CYCLES;
 	((base)->object->id.lib == NULL) &&                                       \
 	(((base)->object->restrictflag & OB_RESTRICT_VIEW) == 0))
 #define BASE_SELECTABLE(v3d, base)  (                                         \
-	(base->lay & v3d->lay) &&                                                 \
+	(v3d != NULL) &&                                                          \
 	(base->object->restrictflag & (OB_RESTRICT_SELECT | OB_RESTRICT_VIEW)) == 0)
 #define BASE_VISIBLE(v3d, base)  (                                            \
 	(base->lay & v3d->lay) &&                                                 \
@@ -1943,6 +1944,8 @@ extern const char *RE_engine_id_CYCLES;
 #define LASTBASE		scene->base.last
 #define BASACT			(scene->basact)
 #define OBACT			(BASACT ? BASACT->object: NULL)
+#define BASACT_NEW		(sl->basact)
+#define OBACT_NEW		(BASACT_NEW ? BASACT_NEW->object: NULL)
 
 #define V3D_CAMERA_LOCAL(v3d) ((!(v3d)->scenelock && (v3d)->camera) ? (v3d)->camera : NULL)
 #define V3D_CAMERA_SCENE(scene, v3d) ((!(v3d)->scenelock && (v3d)->camera) ? (v3d)->camera : (scene)->camera)
@@ -1958,7 +1961,7 @@ extern const char *RE_engine_id_CYCLES;
 #define TIME2FRA(a)     ((((double) scene->r.frs_sec) * (double)(a)) / (double)scene->r.frs_sec_base)
 #define FPS              (((double) scene->r.frs_sec) / (double)scene->r.frs_sec_base)
 
-/* base->flag is in DNA_object_types.h */
+/* base->legacy_flag is in DNA_object_types.h */
 
 /* toolsettings->snap_flag */
 #define SCE_SNAP				1
diff --git a/source/blender/makesdna/DNA_space_types.h b/source/blender/makesdna/DNA_space_types.h
index 5e015544dc9..14eb93e7f23 100644
--- a/source/blender/makesdna/DNA_space_types.h
+++ b/source/blender/makesdna/DNA_space_types.h
@@ -181,7 +181,8 @@ typedef enum eSpaceButtons_Context {
 	BCONTEXT_CONSTRAINT = 11,
 	BCONTEXT_BONE_CONSTRAINT = 12,
 	BCONTEXT_RENDER_LAYER = 13,
-	
+	BCONTEXT_COLLECTION = 14,
+
 	/* always as last... */
 	BCONTEXT_TOT
 } eSpaceButtons_Context;
@@ -1343,6 +1344,20 @@ typedef enum eSpaceClip_GPencil_Source {
 	SC_GPENCIL_SRC_TRACK = 1,
 } eSpaceClip_GPencil_Source;
 
+/* Collection Manager ======================================= */
+
+typedef struct SpaceCollections {
+	SpaceLink *next, *prev;
+	ListBase regionbase;        /* storage of regions for inactive spaces */
+	int spacetype;
+	int flag; /* eSpaceCollections_Flag */
+} SpaceCollections;
+
+/* SpaceClip->flag */
+typedef enum eSpaceCollections_Flag {
+	SC_COLLECTION_DATA_REFRESH = (1 << 0), /* recreate/update SpaceCollections layer data, needed for undo/read/write */
+} eSpaceCollections_Flag;
+
 /* **************** SPACE DEFINES ********************* */
 
 /* space types, moved from DNA_screen_types.h */
@@ -1372,8 +1387,9 @@ typedef enum eSpace_Type {
 	SPACE_CONSOLE  = 18,
 	SPACE_USERPREF = 19,
 	SPACE_CLIP     = 20,
-	
-	SPACEICONMAX = SPACE_CLIP
+	SPACE_COLLECTIONS   = 21,
+
+	SPACEICONMAX = SPACE_COLLECTIONS
 } eSpace_Type;
 
 /* use for function args */
diff --git a/source/blender/makesdna/DNA_userdef_types.h b/source/blender/makesdna/DNA_userdef_types.h
index 1dc6c7ab578..6cc24f74cee 100644
--- a/source/blender/makesdna/DNA_userdef_types.h
+++ b/source/blender/makesdna/DNA_userdef_types.h
@@ -390,7 +390,8 @@ typedef struct bTheme {
 	ThemeSpace tuserpref;
 	ThemeSpace tconsole;
 	ThemeSpace tclip;
-	
+	ThemeSpace tcollections;
+
 	/* 20 sets of bone colors for this theme */
 	ThemeWireColor tarm[20];
 	/*ThemeWireColor tobj[20];*/
diff --git a/source/blender/makesdna/intern/makesdna.c b/source/blender/makesdna/intern/makesdna.c
index 2cea8715a65..b6cf3d555fa 100644
--- a/source/blender/makesdna/intern/makesdna.c
+++ b/source/blender/makesdna/intern/makesdna.c
@@ -130,6 +130,7 @@ static const char *includefiles[] = {
 	"DNA_freestyle_types.h",
 	"DNA_linestyle_types.h",
 	"DNA_cachefile_types.h",
+	"DNA_layer_types.h",
 	/* see comment above before editing! */
 
 	/* empty string to indicate end of includefiles */
@@ -1342,4 +1343,5 @@ int main(int argc, char **argv)
 #include "DNA_freestyle_types.h"
 #include "DNA_linestyle_types.h"
 #include "DNA_cachefile_types.h"
+#include "DNA_layer_types.h"
 /* end of list */