Fix #33131: more useful cycles behavior for layers that are both marked excluded

and as mask layer. Now it will still use the mask, and have no further influence
rather than just being excluded entirely.

4 files changed, 25 insertions, 3 deletions

diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index a18cb79c102..67255d9f707 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -253,6 +253,7 @@ Object *BlenderSync::sync_object(BL::Object b_parent, int persistent_id[OBJECT_P
 	if(use_holdout != object->use_holdout) {
 		object->use_holdout = use_holdout;
 		scene->object_manager->tag_update(scene);
+		object_updated = true;
 	}
 
 	/* object sync
@@ -278,6 +279,10 @@ Object *BlenderSync::sync_object(BL::Object b_parent, int persistent_id[OBJECT_P
 			object->random_id ^= hash_int(hash_string(b_parent.name().c_str()));
 		}
 
+		/* make holdout objects on excluded layer invisible for non-camera rays */
+		if(use_holdout && (layer_flag & render_layer.exclude_layer))
+			object->visibility &= ~(PATH_RAY_ALL - PATH_RAY_CAMERA);
+
 		/* camera flag is not actually used, instead is tested
 		 * against render layer flags */
 		if(object->visibility & PATH_RAY_CAMERA) {
diff --git a/intern/cycles/blender/blender_sync.cpp b/intern/cycles/blender/blender_sync.cpp
index eacfbdadba0..ce589f55e60 100644
--- a/intern/cycles/blender/blender_sync.cpp
+++ b/intern/cycles/blender/blender_sync.cpp
@@ -241,6 +241,7 @@ void BlenderSync::sync_render_layers(BL::SpaceView3D b_v3d, const char *layer)
 			render_layer.use_localview = (b_v3d.local_view() ? true : false);
 			render_layer.scene_layer = get_layer(b_v3d.layers(), b_v3d.layers_local_view(), render_layer.use_localview);
 			render_layer.layer = render_layer.scene_layer;
+			render_layer.exclude_layer = 0;
 			render_layer.holdout_layer = 0;
 			render_layer.material_override = PointerRNA_NULL;
 			render_layer.use_background = true;
@@ -258,10 +259,16 @@ void BlenderSync::sync_render_layers(BL::SpaceView3D b_v3d, const char *layer)
 	for(r.layers.begin(b_rlay); b_rlay != r.layers.end(); ++b_rlay) {
 		if((!layer && first_layer) || (layer && b_rlay->name() == layer)) {
 			render_layer.name = b_rlay->name();
-			render_layer.scene_layer = get_layer(b_scene.layers()) & ~get_layer(b_rlay->layers_exclude());
-			render_layer.layer = get_layer(b_rlay->layers());
+
 			render_layer.holdout_layer = get_layer(b_rlay->layers_zmask());
+			render_layer.exclude_layer = get_layer(b_rlay->layers_exclude());
+
+			render_layer.scene_layer = get_layer(b_scene.layers()) & ~render_layer.exclude_layer;
+			render_layer.scene_layer |= render_layer.exclude_layer & render_layer.holdout_layer;
+
+			render_layer.layer = get_layer(b_rlay->layers());
 			render_layer.layer |= render_layer.holdout_layer;
+
 			render_layer.material_override = b_rlay->material_override();
 			render_layer.use_background = b_rlay->use_sky();
 			render_layer.use_viewport_visibility = false;
diff --git a/intern/cycles/blender/blender_sync.h b/intern/cycles/blender/blender_sync.h
index 9a478118c04..71781bc5459 100644
--- a/intern/cycles/blender/blender_sync.h
+++ b/intern/cycles/blender/blender_sync.h
@@ -116,7 +116,8 @@ private:
 
 	struct RenderLayerInfo {
 		RenderLayerInfo()
-		: scene_layer(0), layer(0), holdout_layer(0),
+		: scene_layer(0), layer(0),
+		  holdout_layer(0), exclude_layer(0),
 		  material_override(PointerRNA_NULL),
 		  use_background(true),
 		  use_viewport_visibility(false),
@@ -127,6 +128,7 @@ private:
 		uint scene_layer;
 		uint layer;
 		uint holdout_layer;
+		uint exclude_layer;
 		BL::Material material_override;
 		bool use_background;
 		bool use_viewport_visibility;
diff --git a/intern/cycles/render/light.cpp b/intern/cycles/render/light.cpp
index d77516a1b18..4173da453fd 100644
--- a/intern/cycles/render/light.cpp
+++ b/intern/cycles/render/light.cpp
@@ -147,6 +147,10 @@ void LightManager::device_update_distribution(Device *device, DeviceScene *dscen
 		Mesh *mesh = object->mesh;
 		bool have_emission = false;
 
+		/* skip if we are not visible for BSDFs */
+		if(!(object->visibility & (PATH_RAY_DIFFUSE|PATH_RAY_GLOSSY|PATH_RAY_TRANSMIT)))
+			continue;
+
 		/* skip if we have no emission shaders */
 		foreach(uint sindex, mesh->used_shaders) {
 			Shader *shader = scene->shaders[sindex];
@@ -183,6 +187,10 @@ void LightManager::device_update_distribution(Device *device, DeviceScene *dscen
 		Mesh *mesh = object->mesh;
 		bool have_emission = false;
 
+		/* skip if we are not visible for BSDFs */
+		if(!(object->visibility & (PATH_RAY_DIFFUSE|PATH_RAY_GLOSSY|PATH_RAY_TRANSMIT)))
+			continue;
+
 		/* skip if we have no emission shaders */
 		foreach(uint sindex, mesh->used_shaders) {
 			Shader *shader = scene->shaders[sindex];