Render: move Cycles visibility, holdout and shadow catcher properties to Blender

The immediate reason for this is that we want to be able to initialize them
to different defaults for light objects, which is hard with Python properties.
But in general it is useful to be able to share these with other renderers.

As a side effect, Eevee now supports a per-object holdout instead of only
per-collection.

Differential Revision: https://developer.blender.org/D12133

10 files changed, 129 insertions, 48 deletions

diff --git a/intern/cycles/blender/addon/properties.py b/intern/cycles/blender/addon/properties.py
index 70efb1054a2..124223635d1 100644
--- a/intern/cycles/blender/addon/properties.py
+++ b/intern/cycles/blender/addon/properties.py
@@ -1164,12 +1164,6 @@ class CyclesVisibilitySettings(bpy.types.PropertyGroup):
 
     @classmethod
     def register(cls):
-        bpy.types.Object.cycles_visibility = PointerProperty(
-            name="Cycles Visibility Settings",
-            description="Cycles visibility settings",
-            type=cls,
-        )
-
         bpy.types.World.cycles_visibility = PointerProperty(
             name="Cycles Visibility Settings",
             description="Cycles visibility settings",
@@ -1178,7 +1172,6 @@ class CyclesVisibilitySettings(bpy.types.PropertyGroup):
 
     @classmethod
     def unregister(cls):
-        del bpy.types.Object.cycles_visibility
         del bpy.types.World.cycles_visibility
 
 
@@ -1276,20 +1269,6 @@ class CyclesObjectSettings(bpy.types.PropertyGroup):
         subtype='DISTANCE',
     )
 
-    is_shadow_catcher: BoolProperty(
-        name="Shadow Catcher",
-        description="Only render shadows on this object, for compositing renders into real footage",
-        default=False,
-    )
-
-    is_holdout: BoolProperty(
-        name="Holdout",
-        description="Render objects as a holdout or matte, creating a "
-        "hole in the image with zero alpha, to fill out in "
-        "compositing with real footage or another render",
-        default=False,
-    )
-
     @classmethod
     def register(cls):
         bpy.types.Object.cycles = PointerProperty(
diff --git a/intern/cycles/blender/addon/ui.py b/intern/cycles/blender/addon/ui.py
index e804f697571..47f7b4c6d73 100644
--- a/intern/cycles/blender/addon/ui.py
+++ b/intern/cycles/blender/addon/ui.py
@@ -1270,10 +1270,9 @@ class CYCLES_OBJECT_PT_visibility(CyclesButtonsPanel, Panel):
         col.prop(ob, "hide_render", text="Renders", invert_checkbox=True, toggle=False)
 
         if has_geometry_visibility(ob):
-            cob = ob.cycles
             col = layout.column(heading="Mask")
-            col.prop(cob, "is_shadow_catcher")
-            col.prop(cob, "is_holdout")
+            col.prop(ob, "is_shadow_catcher")
+            col.prop(ob, "is_holdout")
 
 
 class CYCLES_OBJECT_PT_visibility_ray_visibility(CyclesButtonsPanel, Panel):
@@ -1293,19 +1292,17 @@ class CYCLES_OBJECT_PT_visibility_ray_visibility(CyclesButtonsPanel, Panel):
 
         scene = context.scene
         ob = context.object
-        cob = ob.cycles
-        visibility = ob.cycles_visibility
 
         col = layout.column()
-        col.prop(visibility, "camera")
-        col.prop(visibility, "diffuse")
-        col.prop(visibility, "glossy")
-        col.prop(visibility, "transmission")
-        col.prop(visibility, "scatter")
+        col.prop(ob, "visible_camera", text="Camera")
+        col.prop(ob, "visible_diffuse", text="Diffuse")
+        col.prop(ob, "visible_glossy", text="Glossy")
+        col.prop(ob, "visible_transmission", text="Transmission")
+        col.prop(ob, "visible_volume_scatter", text="Volume Scatter")
 
         if ob.type != 'LIGHT':
             sub = col.column()
-            sub.prop(visibility, "shadow")
+            sub.prop(ob, "visible_shadow", text="Shadow")
 
 
 class CYCLES_OBJECT_PT_visibility_culling(CyclesButtonsPanel, Panel):
diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index 65b5ac2c58f..4711e0cbe76 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -199,8 +199,7 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
 
   /* Visibility flags for both parent and child. */
   PointerRNA cobject = RNA_pointer_get(&b_ob.ptr, "cycles");
-  bool use_holdout = get_boolean(cobject, "is_holdout") ||
-                     b_parent.holdout_get(PointerRNA_NULL, b_view_layer);
+  bool use_holdout = b_parent.holdout_get(PointerRNA_NULL, b_view_layer);
   uint visibility = object_ray_visibility(b_ob) & PATH_RAY_ALL_VISIBILITY;
 
   if (b_parent.ptr.data != b_ob.ptr.data) {
@@ -287,8 +286,7 @@ Object *BlenderSync::sync_object(BL::Depsgraph &b_depsgraph,
 
   object->set_visibility(visibility);
 
-  bool is_shadow_catcher = get_boolean(cobject, "is_shadow_catcher");
-  object->set_is_shadow_catcher(is_shadow_catcher);
+  object->set_is_shadow_catcher(b_ob.is_shadow_catcher());
 
   float shadow_terminator_shading_offset = get_float(cobject, "shadow_terminator_offset");
   object->set_shadow_terminator_shading_offset(shadow_terminator_shading_offset);
diff --git a/intern/cycles/blender/blender_util.h b/intern/cycles/blender/blender_util.h
index 43dbb4105df..2b2188b023d 100644
--- a/intern/cycles/blender/blender_util.h
+++ b/intern/cycles/blender/blender_util.h
@@ -596,15 +596,14 @@ static inline Mesh::SubdivisionType object_subdivision_type(BL::Object &b_ob,
 
 static inline uint object_ray_visibility(BL::Object &b_ob)
 {
-  PointerRNA cvisibility = RNA_pointer_get(&b_ob.ptr, "cycles_visibility");
   uint flag = 0;
 
-  flag |= get_boolean(cvisibility, "camera") ? PATH_RAY_CAMERA : 0;
-  flag |= get_boolean(cvisibility, "diffuse") ? PATH_RAY_DIFFUSE : 0;
-  flag |= get_boolean(cvisibility, "glossy") ? PATH_RAY_GLOSSY : 0;
-  flag |= get_boolean(cvisibility, "transmission") ? PATH_RAY_TRANSMIT : 0;
-  flag |= get_boolean(cvisibility, "shadow") ? PATH_RAY_SHADOW : 0;
-  flag |= get_boolean(cvisibility, "scatter") ? PATH_RAY_VOLUME_SCATTER : 0;
+  flag |= b_ob.visible_camera() ? PATH_RAY_CAMERA : 0;
+  flag |= b_ob.visible_diffuse() ? PATH_RAY_DIFFUSE : 0;
+  flag |= b_ob.visible_glossy() ? PATH_RAY_GLOSSY : 0;
+  flag |= b_ob.visible_transmission() ? PATH_RAY_TRANSMIT : 0;
+  flag |= b_ob.visible_shadow() ? PATH_RAY_SHADOW : 0;
+  flag |= b_ob.visible_volume_scatter() ? PATH_RAY_VOLUME_SCATTER : 0;
 
   return flag;
 }
diff --git a/release/scripts/startup/bl_ui/properties_object.py b/release/scripts/startup/bl_ui/properties_object.py
index 4ea1ec26738..46a16a70dca 100644
--- a/release/scripts/startup/bl_ui/properties_object.py
+++ b/release/scripts/startup/bl_ui/properties_object.py
@@ -399,6 +399,10 @@ class OBJECT_PT_visibility(ObjectButtonsPanel, Panel):
             col = layout.column(heading="Grease Pencil")
             col.prop(ob, "use_grease_pencil_lights", toggle=False)
 
+        layout.separator()
+        col = layout.column(heading="Mask")
+        col.prop(ob, "is_holdout")
+
 
 class OBJECT_PT_custom_props(ObjectButtonsPanel, PropertyPanel, Panel):
     COMPAT_ENGINES = {'BLENDER_RENDER', 'BLENDER_EEVEE', 'BLENDER_WORKBENCH'}
diff --git a/source/blender/blenkernel/intern/layer.c b/source/blender/blenkernel/intern/layer.c
index 5940569ba76..e7d83c668c8 100644
--- a/source/blender/blenkernel/intern/layer.c
+++ b/source/blender/blenkernel/intern/layer.c
@@ -1020,7 +1020,7 @@ static void layer_collection_objects_sync(ViewLayer *view_layer,
     }
 
     /* Holdout and indirect only */
-    if (layer->flag & LAYER_COLLECTION_HOLDOUT) {
+    if ((layer->flag & LAYER_COLLECTION_HOLDOUT) || (base->object->visibility_flag & OB_HOLDOUT)) {
       base->flag_from_collection |= BASE_HOLDOUT;
     }
     if (layer->flag & LAYER_COLLECTION_INDIRECT_ONLY) {
diff --git a/source/blender/blenkernel/intern/object.c b/source/blender/blenkernel/intern/object.c
index 6a309e040c0..f969ca0ff1c 100644
--- a/source/blender/blenkernel/intern/object.c
+++ b/source/blender/blenkernel/intern/object.c
@@ -2081,6 +2081,12 @@ static void object_init(Object *ob, const short ob_type)
   if (ob->type == OB_GPENCIL) {
     ob->dtx |= OB_USE_GPENCIL_LIGHTS;
   }
+
+  if (ob->type == OB_LAMP) {
+    /* Lights are invisible to camera rays and are assumed to be a
+     * shadow catcher by default. */
+    ob->visibility_flag |= OB_HIDE_CAMERA | OB_SHADOW_CATCHER;
+  }
 }
 
 void *BKE_object_obdata_add_from_type(Main *bmain, int type, const char *name)
diff --git a/source/blender/blenloader/intern/versioning_cycles.c b/source/blender/blenloader/intern/versioning_cycles.c
index 94ee89c5120..90e6b43f02e 100644
--- a/source/blender/blenloader/intern/versioning_cycles.c
+++ b/source/blender/blenloader/intern/versioning_cycles.c
@@ -78,6 +78,12 @@ static IDProperty *cycles_properties_from_ID(ID *id)
   return (idprop) ? IDP_GetPropertyTypeFromGroup(idprop, "cycles", IDP_GROUP) : NULL;
 }
 
+static IDProperty *cycles_visibility_properties_from_ID(ID *id)
+{
+  IDProperty *idprop = IDP_GetProperties(id, false);
+  return (idprop) ? IDP_GetPropertyTypeFromGroup(idprop, "cycles_visibility", IDP_GROUP) : NULL;
+}
+
 static IDProperty *cycles_properties_from_view_layer(ViewLayer *view_layer)
 {
   IDProperty *idprop = view_layer->id_properties;
@@ -1600,4 +1606,35 @@ void do_versions_after_linking_cycles(Main *bmain)
       }
     }
   }
+
+  /* Move visibility from Cycles to Blender. */
+  if (!MAIN_VERSION_ATLEAST(bmain, 300, 17)) {
+    LISTBASE_FOREACH (Object *, object, &bmain->objects) {
+      IDProperty *cvisibility = cycles_visibility_properties_from_ID(&object->id);
+      int flag = 0;
+
+      if (cvisibility) {
+        flag |= cycles_property_boolean(cvisibility, "camera", true) ? 0 : OB_HIDE_CAMERA;
+        flag |= cycles_property_boolean(cvisibility, "diffuse", true) ? 0 : OB_HIDE_DIFFUSE;
+        flag |= cycles_property_boolean(cvisibility, "glossy", true) ? 0 : OB_HIDE_GLOSSY;
+        flag |= cycles_property_boolean(cvisibility, "transmission", true) ? 0 :
+                                                                             OB_HIDE_TRANSMISSION;
+        flag |= cycles_property_boolean(cvisibility, "scatter", true) ? 0 : OB_HIDE_VOLUME_SCATTER;
+        flag |= cycles_property_boolean(cvisibility, "shadow", true) ? 0 : OB_HIDE_SHADOW;
+      }
+
+      IDProperty *cobject = cycles_properties_from_ID(&object->id);
+      if (cobject) {
+        flag |= cycles_property_boolean(cobject, "is_holdout", false) ? OB_HOLDOUT : 0;
+        flag |= cycles_property_boolean(cobject, "is_shadow_catcher", false) ? OB_SHADOW_CATCHER :
+                                                                               0;
+      }
+
+      if (object->type == OB_LAMP) {
+        flag |= OB_HIDE_CAMERA | OB_SHADOW_CATCHER;
+      }
+
+      object->visibility_flag |= flag;
+    }
+  }
 }
diff --git a/source/blender/makesdna/DNA_object_types.h b/source/blender/makesdna/DNA_object_types.h
index a37536f7022..e7091c78f71 100644
--- a/source/blender/makesdna/DNA_object_types.h
+++ b/source/blender/makesdna/DNA_object_types.h
@@ -385,14 +385,14 @@ typedef struct Object {
   short softflag;
 
   /** For restricting view, select, render etc. accessible in outliner. */
-  char visibility_flag;
+  short visibility_flag;
 
-  /** Flag for pinning. */
-  char shapeflag;
   /** Current shape key for menu or pinned. */
   short shapenr;
+  /** Flag for pinning. */
+  char shapeflag;
 
-  char _pad3[2];
+  char _pad3[1];
 
   /** Object constraints. */
   ListBase constraints;
@@ -675,6 +675,14 @@ enum {
   OB_HIDE_VIEWPORT = 1 << 0,
   OB_HIDE_SELECT = 1 << 1,
   OB_HIDE_RENDER = 1 << 2,
+  OB_HIDE_CAMERA = 1 << 3,
+  OB_HIDE_DIFFUSE = 1 << 4,
+  OB_HIDE_GLOSSY = 1 << 5,
+  OB_HIDE_TRANSMISSION = 1 << 6,
+  OB_HIDE_VOLUME_SCATTER = 1 << 7,
+  OB_HIDE_SHADOW = 1 << 8,
+  OB_HOLDOUT = 1 << 9,
+  OB_SHADOW_CATCHER = 1 << 10
 };
 
 /* ob->shapeflag */
diff --git a/source/blender/makesrna/intern/rna_object.c b/source/blender/makesrna/intern/rna_object.c
index f0b83f3eb46..bef430b0314 100644
--- a/source/blender/makesrna/intern/rna_object.c
+++ b/source/blender/makesrna/intern/rna_object.c
@@ -2963,6 +2963,59 @@ static void rna_def_object_visibility(StructRNA *srna)
   RNA_def_property_ui_text(prop, "Display Instancer", "Make instancer visible in the viewport");
   RNA_def_property_update(
       prop, NC_OBJECT | ND_DRAW, "rna_Object_duplicator_visibility_flag_update");
+
+  /* Ray visibility. */
+  prop = RNA_def_property(srna, "visible_camera", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_CAMERA);
+  RNA_def_property_ui_text(prop, "Camera Visibility", "Object visibility to camera rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  prop = RNA_def_property(srna, "visible_diffuse", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_DIFFUSE);
+  RNA_def_property_ui_text(prop, "Diffuse Visibility", "Object visibility to diffuse rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  prop = RNA_def_property(srna, "visible_glossy", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_GLOSSY);
+  RNA_def_property_ui_text(prop, "Glossy Visibility", "Object visibility to glossy rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  prop = RNA_def_property(srna, "visible_transmission", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_TRANSMISSION);
+  RNA_def_property_ui_text(
+      prop, "Transmission Visibility", "Object visibility to transmission rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  prop = RNA_def_property(srna, "visible_volume_scatter", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_VOLUME_SCATTER);
+  RNA_def_property_ui_text(
+      prop, "Volume Scatter Visibility", "Object visibility to volume scattering rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  prop = RNA_def_property(srna, "visible_shadow", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_negative_sdna(prop, NULL, "visibility_flag", OB_HIDE_SHADOW);
+  RNA_def_property_ui_text(prop, "Shadow Visibility", "Object visibility to shadow rays");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
+
+  /* Holdout and shadow catcher. */
+  prop = RNA_def_property(srna, "is_holdout", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "visibility_flag", OB_HOLDOUT);
+  RNA_def_property_ui_text(
+      prop,
+      "Holdout",
+      "Render objects as a holdout or matte, creating a hole in the image with zero alpha, to "
+      "fill out in compositing with real footage or another render");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_hide_update");
+
+  prop = RNA_def_property(srna, "is_shadow_catcher", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "visibility_flag", OB_SHADOW_CATCHER);
+  RNA_def_property_ui_text(
+      prop,
+      "Shadow Catcher",
+      "Only render shadows and reflections on this object, for compositing renders into real "
+      "footage. Objects with this setting are considered to already exist in the footage, "
+      "objects without it are synthetic objects being composited into it");
+  RNA_def_property_update(prop, NC_OBJECT | ND_DRAW, "rna_Object_internal_update_draw");
 }
 
 static void rna_def_object(BlenderRNA *brna)