SceneRenderLayer Removal/Refactor

This patch moves all the functionality previously in SceneRenderLayer to SceneLayer.
If we want to rename some of these structs now would be a good time to do it, before they are in SceneLayer.

Everything should be working, though I will test things further tomorrow. Once this is committed depsgraph can get
rid of the workaround added in rna_Main_meshes_new_from_object and finish whatever this patch was preventing from being finished.

This patch also adds a few placeholders for the overrides (samples, ...). These are obviously not working, so some unittests that rely on 'lay', and 'zmask' will fail.

This patch does not addressed the change of moving samples to ViewRender (I have this as a separate patch and needs some separate discussion).

Following next is the individual note of the individual parts that were committed.

Note 1: It is up to Cycles to still get rid of exclude_layer internally.
Note 2: Cycles still need to handle its own doversion for the use_layer_samples cases and

(1) Remove the override as it is
(2) Add a new override (scene.cycles.samples) if scene.cycles.use_layer_samples != IGNORE

Respecting the expected behaviour when scene.cycles.use_layer_samples == BOUNDED.

Note 3: Cycles still need to implement the per-object holdout
(similar to how we do shadow catcher).

Note 4: There are parts of the old (Blender Internal) rendering pipeline that is still
using lay, e.g., in shi->lay.

Honestly it will be easier to purge the entire Blender Internal code away instead of taking things from it bit by bit.

Reviewers: sergey, campbellbarton, brecht

Differential Revision: https://developer.blender.org/D2919

3 files changed, 57 insertions, 54 deletions

diff --git a/source/blender/editors/render/render_internal.c b/source/blender/editors/render/render_internal.c
index 3f2951b2a3e..701227091b3 100644
--- a/source/blender/editors/render/render_internal.c
+++ b/source/blender/editors/render/render_internal.c
@@ -108,7 +108,6 @@ typedef struct RenderJob {
 	 */
 	Depsgraph *depsgraph;
 	Render *re;
-	SceneRenderLayer *srl;
 	struct Object *camera_override;
 	int lay_override;
 	bool v3d_override;
@@ -261,7 +260,7 @@ static void image_buffer_rect_update(RenderJob *rj, RenderResult *rr, ImBuf *ibu
 /* set callbacks, exported to sequence render too.
  * Only call in foreground (UI) renders. */
 
-static void screen_render_scene_layer_set(wmOperator *op, Main *mainp, Scene **scene, SceneRenderLayer **srl)
+static void screen_render_scene_layer_set(wmOperator *op, Main *mainp, Scene **scene, SceneLayer **scene_layer)
 {
 	/* single layer re-render */
 	if (RNA_struct_property_is_set(op->ptr, "scene")) {
@@ -281,14 +280,14 @@ static void screen_render_scene_layer_set(wmOperator *op, Main *mainp, Scene **s
 	}
 
 	if (RNA_struct_property_is_set(op->ptr, "layer")) {
-		SceneRenderLayer *rl;
+		SceneLayer *rl;
 		char rl_name[RE_MAXNAME];
 
 		RNA_string_get(op->ptr, "layer", rl_name);
-		rl = (SceneRenderLayer *)BLI_findstring(&(*scene)->r.layers, rl_name, offsetof(SceneRenderLayer, name));
+		rl = (SceneLayer *)BLI_findstring(&(*scene)->render_layers, rl_name, offsetof(SceneLayer, name));
 		
 		if (rl)
-			*srl = rl;
+			*scene_layer = rl;
 	}
 }
 
@@ -296,9 +295,8 @@ static void screen_render_scene_layer_set(wmOperator *op, Main *mainp, Scene **s
 static int screen_render_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneLayer *scene_layer = CTX_data_scene_layer(C);
+	SceneLayer *scene_layer = NULL;
 	Depsgraph *depsgraph = CTX_data_depsgraph(C);
-	SceneRenderLayer *srl = NULL;
 	Render *re;
 	Image *ima;
 	View3D *v3d = CTX_wm_view3d(C);
@@ -309,7 +307,10 @@ static int screen_render_exec(bContext *C, wmOperator *op)
 	struct Object *camera_override = v3d ? V3D_CAMERA_LOCAL(v3d) : NULL;
 
 	/* custom scene and single layer re-render */
-	screen_render_scene_layer_set(op, mainp, &scene, &srl);
+	screen_render_scene_layer_set(op, mainp, &scene, &scene_layer);
+	if (scene_layer == NULL) {
+		scene_layer = CTX_data_scene_layer(C);
+	}
 
 	if (!is_animation && is_write_still && BKE_imtype_is_movie(scene->r.im_format.imtype)) {
 		BKE_report(op->reports, RPT_ERROR, "Cannot write a single file with an animation format selected");
@@ -339,7 +340,7 @@ static int screen_render_exec(bContext *C, wmOperator *op)
 	if (is_animation)
 		RE_BlenderAnim(re, mainp, scene, camera_override, lay_override, scene->r.sfra, scene->r.efra, scene->r.frame_step);
 	else
-		RE_BlenderFrame(re, mainp, scene, srl, camera_override, lay_override, scene->r.cfra, is_write_still);
+		RE_BlenderFrame(re, mainp, scene, scene_layer, camera_override, lay_override, scene->r.cfra, is_write_still);
 	BLI_end_threaded_malloc();
 
 	RE_SetReports(re, NULL);
@@ -616,7 +617,7 @@ static void render_startjob(void *rjv, short *stop, short *do_update, float *pro
 	if (rj->anim)
 		RE_BlenderAnim(rj->re, rj->main, rj->scene, rj->camera_override, rj->lay_override, rj->scene->r.sfra, rj->scene->r.efra, rj->scene->r.frame_step);
 	else
-		RE_BlenderFrame(rj->re, rj->main, rj->scene, rj->srl, rj->camera_override, rj->lay_override, rj->scene->r.cfra, rj->write_still);
+		RE_BlenderFrame(rj->re, rj->main, rj->scene, rj->scene_layer, rj->camera_override, rj->lay_override, rj->scene->r.cfra, rj->write_still);
 
 	RE_SetReports(rj->re, NULL);
 }
@@ -669,7 +670,7 @@ static void render_endjob(void *rjv)
 	/* potentially set by caller */
 	rj->scene->r.scemode &= ~R_NO_FRAME_UPDATE;
 	
-	if (rj->srl) {
+	if (rj->scene_layer) {
 		nodeUpdateID(rj->scene->nodetree, &rj->scene->id);
 		WM_main_add_notifier(NC_NODE | NA_EDITED, rj->scene);
 	}
@@ -699,7 +700,7 @@ static void render_endjob(void *rjv)
 	 * engine API, so lets use simple and robust way for now
 	 *                                          - sergey -
 	 */
-	if (rj->scene->r.layers.first != rj->scene->r.layers.last ||
+	if (rj->scene->render_layers.first != rj->scene->render_layers.last ||
 	    rj->image_outdated)
 	{
 		void *lock;
@@ -844,8 +845,8 @@ static int screen_render_invoke(bContext *C, wmOperator *op, const wmEvent *even
 {
 	/* new render clears all callbacks */
 	Main *mainp;
+	SceneLayer *scene_layer = NULL;
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = NULL;
 	Render *re;
 	wmJob *wm_job;
 	RenderJob *rj;
@@ -912,7 +913,10 @@ static int screen_render_invoke(bContext *C, wmOperator *op, const wmEvent *even
 	jobflag = WM_JOB_EXCL_RENDER | WM_JOB_PRIORITY | WM_JOB_PROGRESS;
 	
 	/* custom scene and single layer re-render */
-	screen_render_scene_layer_set(op, mainp, &scene, &srl);
+	screen_render_scene_layer_set(op, mainp, &scene, &scene_layer);
+	if (scene_layer == NULL) {
+		scene_layer = CTX_data_scene_layer(C);
+	}
 
 	if (RNA_struct_property_is_set(op->ptr, "layer"))
 		jobflag |= WM_JOB_SUSPEND;
@@ -922,10 +926,9 @@ static int screen_render_invoke(bContext *C, wmOperator *op, const wmEvent *even
 	rj->main = mainp;
 	rj->scene = scene;
 	rj->current_scene = rj->scene;
-	rj->scene_layer = CTX_data_scene_layer(C);
+	rj->scene_layer = scene_layer;
 	/* TODO(sergey): Render engine should be using own depsgraph. */
 	rj->depsgraph = CTX_data_depsgraph(C);
-	rj->srl = srl;
 	rj->camera_override = camera_override;
 	rj->lay_override = 0;
 	rj->anim = is_animation;
@@ -1283,10 +1286,10 @@ static void render_view3d_startjob(void *customdata, short *stop, short *do_upda
 		/* initalize always */
 		if (use_border) {
 			rdata.mode |= R_BORDER;
-			RE_InitState(re, NULL, &rdata, &rp->scene->view_render, NULL, rp->ar->winx, rp->ar->winy, &cliprct);
+			RE_InitState(re, NULL, &rdata, &rp->scene->render_layers, rp->scene->active_layer, &rp->scene->view_render, NULL, rp->ar->winx, rp->ar->winy, &cliprct);
 		}
 		else
-			RE_InitState(re, NULL, &rdata, &rp->scene->view_render, NULL, rp->ar->winx, rp->ar->winy, NULL);
+			RE_InitState(re, NULL, &rdata, &rp->scene->render_layers, rp->scene->active_layer, &rp->scene->view_render, NULL, rp->ar->winx, rp->ar->winy, NULL);
 	}
 
 	if (orth)
diff --git a/source/blender/editors/render/render_opengl.c b/source/blender/editors/render/render_opengl.c
index d436c562f19..e0193a0caea 100644
--- a/source/blender/editors/render/render_opengl.c
+++ b/source/blender/editors/render/render_opengl.c
@@ -715,7 +715,7 @@ static bool screen_opengl_render_init(bContext *C, wmOperator *op)
 	oglrender->iuser.ok = 1;
 
 	/* create render result */
-	RE_InitState(oglrender->re, NULL, &scene->r, view_render, NULL, sizex, sizey, NULL);
+	RE_InitState(oglrender->re, NULL, &scene->r, &scene->render_layers, scene->active_layer, view_render, NULL, sizex, sizey, NULL);
 
 	/* create render views */
 	screen_opengl_views_setup(oglrender);
diff --git a/source/blender/editors/render/render_shading.c b/source/blender/editors/render/render_shading.c
index 323b17797a0..a98751176fc 100644
--- a/source/blender/editors/render/render_shading.c
+++ b/source/blender/editors/render/render_shading.c
@@ -772,9 +772,9 @@ static int freestyle_active_module_poll(bContext *C)
 static int freestyle_module_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	BKE_freestyle_module_add(&srl->freestyleConfig);
+	BKE_freestyle_module_add(&scene_layer->freestyle_config);
 
 	WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
 
@@ -798,11 +798,11 @@ void SCENE_OT_freestyle_module_add(wmOperatorType *ot)
 static int freestyle_module_remove_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 	PointerRNA ptr = CTX_data_pointer_get_type(C, "freestyle_module", &RNA_FreestyleModuleSettings);
 	FreestyleModuleConfig *module = ptr.data;
 
-	BKE_freestyle_module_delete(&srl->freestyleConfig, module);
+	BKE_freestyle_module_delete(&scene_layer->freestyle_config, module);
 
 	DEG_id_tag_update(&scene->id, 0);
 	WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
@@ -828,12 +828,12 @@ void SCENE_OT_freestyle_module_remove(wmOperatorType *ot)
 static int freestyle_module_move_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 	PointerRNA ptr = CTX_data_pointer_get_type(C, "freestyle_module", &RNA_FreestyleModuleSettings);
 	FreestyleModuleConfig *module = ptr.data;
 	int dir = RNA_enum_get(op->ptr, "direction");
 
-	if (BKE_freestyle_module_move(&srl->freestyleConfig, module, dir)) {
+	if (BKE_freestyle_module_move(&scene_layer->freestyle_config, module, dir)) {
 		DEG_id_tag_update(&scene->id, 0);
 		WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
 	}
@@ -870,9 +870,9 @@ static int freestyle_lineset_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Main *bmain = CTX_data_main(C);
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	BKE_freestyle_lineset_add(bmain, &srl->freestyleConfig, NULL);
+	BKE_freestyle_lineset_add(bmain, &scene_layer->freestyle_config, NULL);
 
 	DEG_id_tag_update(&scene->id, 0);
 	WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
@@ -897,21 +897,21 @@ void SCENE_OT_freestyle_lineset_add(wmOperatorType *ot)
 static int freestyle_active_lineset_poll(bContext *C)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	if (!srl) {
+	if (!scene_layer) {
 		return false;
 	}
 
-	return BKE_freestyle_lineset_get_active(&srl->freestyleConfig) != NULL;
+	return BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config) != NULL;
 }
 
 static int freestyle_lineset_copy_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	FRS_copy_active_lineset(&srl->freestyleConfig);
+	FRS_copy_active_lineset(&scene_layer->freestyle_config);
 
 	return OPERATOR_FINISHED;
 }
@@ -934,9 +934,9 @@ void SCENE_OT_freestyle_lineset_copy(wmOperatorType *ot)
 static int freestyle_lineset_paste_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	FRS_paste_active_lineset(&srl->freestyleConfig);
+	FRS_paste_active_lineset(&scene_layer->freestyle_config);
 
 	DEG_id_tag_update(&scene->id, 0);
 	WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
@@ -962,9 +962,9 @@ void SCENE_OT_freestyle_lineset_paste(wmOperatorType *ot)
 static int freestyle_lineset_remove_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 
-	FRS_delete_active_lineset(&srl->freestyleConfig);
+	FRS_delete_active_lineset(&scene_layer->freestyle_config);
 
 	DEG_id_tag_update(&scene->id, 0);
 	WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
@@ -990,10 +990,10 @@ void SCENE_OT_freestyle_lineset_remove(wmOperatorType *ot)
 static int freestyle_lineset_move_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
 	int dir = RNA_enum_get(op->ptr, "direction");
 
-	if (FRS_move_active_lineset(&srl->freestyleConfig, dir)) {
+	if (FRS_move_active_lineset(&scene_layer->freestyle_config, dir)) {
 		DEG_id_tag_update(&scene->id, 0);
 		WM_event_add_notifier(C, NC_SCENE | ND_RENDER_OPTIONS, scene);
 	}
@@ -1030,8 +1030,8 @@ static int freestyle_linestyle_new_exec(bContext *C, wmOperator *op)
 {
 	Main *bmain = CTX_data_main(C);
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 
 	if (!lineset) {
 		BKE_report(op->reports, RPT_ERROR, "No active lineset to add a new line style to");
@@ -1068,8 +1068,8 @@ void SCENE_OT_freestyle_linestyle_new(wmOperatorType *ot)
 static int freestyle_color_modifier_add_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	int type = RNA_enum_get(op->ptr, "type");
 
 	if (!freestyle_linestyle_check_report(lineset, op->reports)) {
@@ -1108,8 +1108,8 @@ void SCENE_OT_freestyle_color_modifier_add(wmOperatorType *ot)
 static int freestyle_alpha_modifier_add_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	int type = RNA_enum_get(op->ptr, "type");
 
 	if (!freestyle_linestyle_check_report(lineset, op->reports)) {
@@ -1148,8 +1148,8 @@ void SCENE_OT_freestyle_alpha_modifier_add(wmOperatorType *ot)
 static int freestyle_thickness_modifier_add_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	int type = RNA_enum_get(op->ptr, "type");
 
 	if (!freestyle_linestyle_check_report(lineset, op->reports)) {
@@ -1188,8 +1188,8 @@ void SCENE_OT_freestyle_thickness_modifier_add(wmOperatorType *ot)
 static int freestyle_geometry_modifier_add_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	int type = RNA_enum_get(op->ptr, "type");
 
 	if (!freestyle_linestyle_check_report(lineset, op->reports)) {
@@ -1241,8 +1241,8 @@ static int freestyle_get_modifier_type(PointerRNA *ptr)
 static int freestyle_modifier_remove_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	PointerRNA ptr = CTX_data_pointer_get_type(C, "modifier", &RNA_LineStyleModifier);
 	LineStyleModifier *modifier = ptr.data;
 
@@ -1291,8 +1291,8 @@ void SCENE_OT_freestyle_modifier_remove(wmOperatorType *ot)
 static int freestyle_modifier_copy_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	PointerRNA ptr = CTX_data_pointer_get_type(C, "modifier", &RNA_LineStyleModifier);
 	LineStyleModifier *modifier = ptr.data;
 
@@ -1341,8 +1341,8 @@ void SCENE_OT_freestyle_modifier_copy(wmOperatorType *ot)
 static int freestyle_modifier_move_exec(bContext *C, wmOperator *op)
 {
 	Scene *scene = CTX_data_scene(C);
-	SceneRenderLayer *srl = BLI_findlink(&scene->r.layers, scene->r.actlay);
-	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&srl->freestyleConfig);
+	SceneLayer *scene_layer = BLI_findlink(&scene->render_layers, scene->active_layer);
+	FreestyleLineSet *lineset = BKE_freestyle_lineset_get_active(&scene_layer->freestyle_config);
 	PointerRNA ptr = CTX_data_pointer_get_type(C, "modifier", &RNA_LineStyleModifier);
 	LineStyleModifier *modifier = ptr.data;
 	int dir = RNA_enum_get(op->ptr, "direction");