Cycles: add per layer collection indirectly on setting.

In the outliner, right click > view layer > set indirect only. This is
like clearing camera ray visibility on objects in the collection, and is
temporary until we have more general dynamic overrides.

10 files changed, 93 insertions, 26 deletions

diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index fd8790c0c79..42e2198779e 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -361,12 +361,11 @@ Object *BlenderSync::sync_object(BL::Depsgraph& b_depsgraph,
 	}
 #endif
 
-	/* TODO: hide objects not on render layer from camera rays. */
-#if 0
-	if(!(layer_flag & view_layer.layer)) {
+	/* Clear camera visibility for indirect only objects. */
+	bool use_indirect_only = b_ob.indirect_only_get(b_view_layer);
+	if(use_indirect_only) {
 		visibility &= ~PATH_RAY_CAMERA;
 	}
-#endif
 
 	/* Don't export completely invisible objects. */
 	if(visibility == 0) {
diff --git a/release/scripts/startup/bl_ui/space_outliner.py b/release/scripts/startup/bl_ui/space_outliner.py
index 284a49c8789..20ef5ae0c53 100644
--- a/release/scripts/startup/bl_ui/space_outliner.py
+++ b/release/scripts/startup/bl_ui/space_outliner.py
@@ -146,6 +146,9 @@ class OUTLINER_MT_collection_view_layer(Menu):
         layout.operator("outliner.collection_exclude_clear")
 
         if context.engine == 'CYCLES':
+            layout.operator("outliner.collection_indirect_only_set")
+            layout.operator("outliner.collection_indirect_only_clear")
+
             layout.operator("outliner.collection_holdout_set")
             layout.operator("outliner.collection_holdout_clear")
 
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index 73a217b8865..1396ad1f97c 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -715,10 +715,13 @@ static int layer_collection_sync(
 				lc->runtime_flag |= LAYER_COLLECTION_HAS_VISIBLE_OBJECTS;
 			}
 
-			/* Holdout */
+			/* Holdout and indirect only */
 			if (lc->flag & LAYER_COLLECTION_HOLDOUT) {
 				base->flag |= BASE_HOLDOUT;
 			}
+			if (lc->flag & LAYER_COLLECTION_INDIRECT_ONLY) {
+				base->flag |= BASE_INDIRECT_ONLY;
+			}
 
 			lc->runtime_flag |= LAYER_COLLECTION_HAS_OBJECTS;
 		}
@@ -760,7 +763,8 @@ void BKE_layer_collection_sync(const Scene *scene, ViewLayer *view_layer)
 		                BASE_SELECTABLE |
 		                BASE_ENABLED_VIEWPORT |
 		                BASE_ENABLED_RENDER |
-		                BASE_HOLDOUT);
+		                BASE_HOLDOUT |
+		                BASE_INDIRECT_ONLY);
 	}
 
 	view_layer->runtime_flag = 0;
diff --git a/source/blender/blenloader/intern/versioning_280.c b/source/blender/blenloader/intern/versioning_280.c
index 94afc410000..068c12daa94 100644
--- a/source/blender/blenloader/intern/versioning_280.c
+++ b/source/blender/blenloader/intern/versioning_280.c
@@ -487,28 +487,15 @@ static void do_version_layers_to_collections(Main *bmain, Scene *scene)
 						nlc->flag |= LAYER_COLLECTION_EXCLUDE;
 					}
 				}
-				else if ((scene->lay & srl->lay & ~(srl->lay_exclude) & (1 << layer)) ||
-				         (srl->lay_zmask & (scene->lay | srl->lay_exclude) & (1 << layer)))
-				{
+				else {
 					if (srl->lay_zmask & (1 << layer)) {
 						have_override = true;
 						lc->flag |= LAYER_COLLECTION_HOLDOUT;
-
-						BKE_override_layer_collection_boolean_add(
-						        lc,
-						        ID_OB,
-						        "cycles.is_holdout",
-						        true);
 					}
 
 					if ((srl->lay & (1 << layer)) == 0) {
 						have_override = true;
-
-						BKE_override_layer_collection_boolean_add(
-						        lc,
-						        ID_OB,
-						        "cycles_visibility.camera",
-						        false);
+						lc->flag |= LAYER_COLLECTION_INDIRECT_ONLY;
 					}
 				}
 			}
diff --git a/source/blender/editors/space_outliner/outliner_collections.c b/source/blender/editors/space_outliner/outliner_collections.c
index eec1d654944..e480cac2dcc 100644
--- a/source/blender/editors/space_outliner/outliner_collections.c
+++ b/source/blender/editors/space_outliner/outliner_collections.c
@@ -625,6 +625,16 @@ static bool collections_holdout_clear_poll(bContext *C)
 	return collections_view_layer_poll(C, true, LAYER_COLLECTION_HOLDOUT);
 }
 
+static bool collections_indirect_only_set_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, false, LAYER_COLLECTION_INDIRECT_ONLY);
+}
+
+static bool collections_indirect_only_clear_poll(bContext *C)
+{
+	return collections_view_layer_poll(C, true, LAYER_COLLECTION_INDIRECT_ONLY);
+}
+
 static void layer_collection_flag_recursive_set(LayerCollection *lc, int flag)
 {
 	for (LayerCollection *nlc = lc->layer_collections.first; nlc; nlc = nlc->next) {
@@ -647,7 +657,9 @@ static int collection_view_layer_exec(bContext *C, wmOperator *op)
 	SpaceOops *soops = CTX_wm_space_outliner(C);
 	struct CollectionEditData data = {.scene = scene, .soops = soops};
 	bool clear = strstr(op->idname, "clear") != NULL;
-	int flag = strstr(op->idname, "holdout") ? LAYER_COLLECTION_HOLDOUT : LAYER_COLLECTION_EXCLUDE;
+	int flag = strstr(op->idname, "holdout") ?       LAYER_COLLECTION_HOLDOUT :
+	           strstr(op->idname, "indirect_only") ? LAYER_COLLECTION_INDIRECT_ONLY :
+	                                                 LAYER_COLLECTION_EXCLUDE;
 
 	data.collections_to_edit = BLI_gset_ptr_new(__func__);
 
@@ -739,6 +751,36 @@ void OUTLINER_OT_collection_holdout_clear(wmOperatorType *ot)
 	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
 }
 
+void OUTLINER_OT_collection_indirect_only_set(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Set Indirect Only";
+	ot->idname = "OUTLINER_OT_collection_indirect_only_set";
+	ot->description = "Set collection to only contribute indirectly (through shadows and reflections) in the view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_indirect_only_set_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
+void OUTLINER_OT_collection_indirect_only_clear(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Clear Indirect Only";
+	ot->idname = "OUTLINER_OT_collection_indirect_only_clear";
+	ot->description = "Clear collection contributing only indirectly in the view layer";
+
+	/* api callbacks */
+	ot->exec = collection_view_layer_exec;
+	ot->poll = collections_indirect_only_clear_poll;
+
+	/* flags */
+	ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
+}
+
 /**
  * Populates the \param objects ListBase with all the outliner selected objects
  * We store it as (Object *)LinkData->data
diff --git a/source/blender/editors/space_outliner/outliner_intern.h b/source/blender/editors/space_outliner/outliner_intern.h
index 9262cf8578c..461d4bd7c56 100644
--- a/source/blender/editors/space_outliner/outliner_intern.h
+++ b/source/blender/editors/space_outliner/outliner_intern.h
@@ -361,6 +361,8 @@ void OUTLINER_OT_collection_exclude_set(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_exclude_clear(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_holdout_set(struct wmOperatorType *ot);
 void OUTLINER_OT_collection_holdout_clear(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_indirect_only_set(struct wmOperatorType *ot);
+void OUTLINER_OT_collection_indirect_only_clear(struct wmOperatorType *ot);
 
 /* outliner_utils.c ---------------------------------------------- */
 
diff --git a/source/blender/editors/space_outliner/outliner_ops.c b/source/blender/editors/space_outliner/outliner_ops.c
index 4bd5a0c3792..576038979d3 100644
--- a/source/blender/editors/space_outliner/outliner_ops.c
+++ b/source/blender/editors/space_outliner/outliner_ops.c
@@ -460,6 +460,8 @@ void outliner_operatortypes(void)
 	WM_operatortype_append(OUTLINER_OT_collection_exclude_clear);
 	WM_operatortype_append(OUTLINER_OT_collection_holdout_set);
 	WM_operatortype_append(OUTLINER_OT_collection_holdout_clear);
+	WM_operatortype_append(OUTLINER_OT_collection_indirect_only_set);
+	WM_operatortype_append(OUTLINER_OT_collection_indirect_only_clear);
 }
 
 static wmKeyMap *outliner_item_drag_drop_modal_keymap(wmKeyConfig *keyconf)
diff --git a/source/blender/makesdna/DNA_layer_types.h b/source/blender/makesdna/DNA_layer_types.h
index 2bfe4c4e13b..e6b1bda7cf5 100644
--- a/source/blender/makesdna/DNA_layer_types.h
+++ b/source/blender/makesdna/DNA_layer_types.h
@@ -105,6 +105,7 @@ enum {
 	BASE_ENABLED_RENDER   = (1 << 7), /* Object is enabled in final render */
 	BASE_ENABLED          = (1 << 9), /* Object is enabled. */
 	BASE_HOLDOUT          = (1 << 10), /* Object masked out from render */
+	BASE_INDIRECT_ONLY    = (1 << 11), /* Object only contributes indirectly to render */
 };
 
 /* LayerCollection->flag */
@@ -115,6 +116,7 @@ enum {
 	/* LAYER_COLLECTION_DEPRECATED3 = (1 << 3), */
 	LAYER_COLLECTION_EXCLUDE = (1 << 4),
 	LAYER_COLLECTION_HOLDOUT = (1 << 5),
+	LAYER_COLLECTION_INDIRECT_ONLY = (1 << 6),
 };
 
 /* Layer Collection->runtime_flag */
diff --git a/source/blender/makesrna/intern/rna_layer.c b/source/blender/makesrna/intern/rna_layer.c
index 1ab69d78f03..d425534fe53 100644
--- a/source/blender/makesrna/intern/rna_layer.c
+++ b/source/blender/makesrna/intern/rna_layer.c
@@ -234,7 +234,13 @@ static void rna_def_layer_collection(BlenderRNA *brna)
 	prop = RNA_def_property(srna, "holdout", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_HOLDOUT);
 	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-	RNA_def_property_ui_text(prop, "Holdout", "Mask out collection from view layer");
+	RNA_def_property_ui_text(prop, "Holdout", "Mask out objects in collection from view layer");
+	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
+
+	prop = RNA_def_property(srna, "indirect_only", PROP_BOOLEAN, PROP_NONE);
+	RNA_def_property_boolean_sdna(prop, NULL, "flag", LAYER_COLLECTION_INDIRECT_ONLY);
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+	RNA_def_property_ui_text(prop, "Indirect Only", "Objects in collection only contribute indirectly (through shadows and reflections) in the view layer");
 	RNA_def_property_update(prop, NC_SCENE | ND_LAYER, "rna_LayerCollection_use_update");
 }
 
diff --git a/source/blender/makesrna/intern/rna_object_api.c b/source/blender/makesrna/intern/rna_object_api.c
index dc799928e5a..44709db9d94 100644
--- a/source/blender/makesrna/intern/rna_object_api.c
+++ b/source/blender/makesrna/intern/rna_object_api.c
@@ -128,7 +128,7 @@ static bool rna_Object_select_get(Object *ob, bContext *C, ReportList *reports)
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
@@ -141,7 +141,7 @@ static bool rna_Object_visible_get(Object *ob, bContext *C, ReportList *reports)
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
@@ -153,13 +153,25 @@ static bool rna_Object_holdout_get(Object *ob, ReportList *reports, ViewLayer *v
 	Base *base = BKE_view_layer_base_find(view_layer, ob);
 
 	if (!base) {
-		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in Render Layer '%s'!", ob->id.name + 2, view_layer->name);
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
 		return -1;
 	}
 
 	return ((base->flag & BASE_HOLDOUT) != 0) ? 1 : 0;
 }
 
+static bool rna_Object_indirect_only_get(Object *ob, ReportList *reports, ViewLayer *view_layer)
+{
+	Base *base = BKE_view_layer_base_find(view_layer, ob);
+
+	if (!base) {
+		BKE_reportf(reports, RPT_ERROR, "Object '%s' not in View Layer '%s'!", ob->id.name + 2, view_layer->name);
+		return -1;
+	}
+
+	return ((base->flag & BASE_INDIRECT_ONLY) != 0) ? 1 : 0;
+}
+
 /* Convert a given matrix from a space to another (using the object and/or a bone as reference). */
 static void rna_Object_mat_convert_space(Object *ob, ReportList *reports, bPoseChannel *pchan,
                                         float *mat, float *mat_ret, int from, int to)
@@ -515,6 +527,14 @@ void RNA_api_object(StructRNA *srna)
 	parm = RNA_def_boolean(func, "result", 0, "", "Object holdout");
 	RNA_def_function_return(func, parm);
 
+	func = RNA_def_function(srna, "indirect_only_get", "rna_Object_indirect_only_get");
+	RNA_def_function_ui_description(func, "Test if object is set to contribute only indirectly (through shadows and reflections) in the view layer");
+	RNA_def_function_flag(func, FUNC_USE_REPORTS);
+	parm = RNA_def_pointer(func, "view_layer", "ViewLayer", "", "");
+	RNA_def_parameter_flags(parm, 0, PARM_REQUIRED);
+	parm = RNA_def_boolean(func, "result", 0, "", "Object indirect only");
+	RNA_def_function_return(func, parm);
+
 	/* Matrix space conversion */
 	func = RNA_def_function(srna, "convert_space", "rna_Object_mat_convert_space");
 	RNA_def_function_ui_description(func, "Convert (transform) the given matrix from one space to another");