Fix inconsistent/broken Cycles object visibility for instances.

Object visibility is now handled by the depsgraph iterator, but this API
was incomplete as it made no distinction for visibility of the object itself,
particles and generated instances.

The depsgraph iterator API now includes information about which part of the
object is visible, and this is used by Cycles to replace the old custom logic.
Cycles and EEVEE visibility should now be consistent, which unfortunately does
means some subtle compatibility breakage for both.

Fixes T58956, T58202, T59284.

Differential Revision: https://developer.blender.org/D4109

7 files changed, 27 insertions, 17 deletions

diff --git a/source/blender/draw/engines/eevee/eevee_engine.c b/source/blender/draw/engines/eevee/eevee_engine.c
index aa2cf3fede3..6a2ca982d53 100644
--- a/source/blender/draw/engines/eevee/eevee_engine.c
+++ b/source/blender/draw/engines/eevee/eevee_engine.c
@@ -130,15 +130,14 @@ void EEVEE_cache_populate(void *vedata, Object *ob)
 	EEVEE_ViewLayerData *sldata = EEVEE_view_layer_data_ensure();
 
 	const DRWContextState *draw_ctx = DRW_context_state_get();
+	const int ob_visibility = DRW_object_visibility_in_active_context(ob);
 	bool cast_shadow = false;
 
-	if (ob->base_flag & BASE_VISIBLE) {
+	if (ob_visibility & OB_VISIBLE_PARTICLES) {
 		EEVEE_hair_cache_populate(vedata, sldata, ob, &cast_shadow);
 	}
 
-	if (DRW_object_is_renderable(ob) &&
-	    DRW_object_is_visible_in_active_context(ob))
-	{
+	if (DRW_object_is_renderable(ob) && (ob_visibility & OB_VISIBLE_SELF)) {
 		if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {
 			EEVEE_materials_cache_populate(vedata, sldata, ob, &cast_shadow);
 		}
diff --git a/source/blender/draw/engines/eevee/eevee_lightcache.c b/source/blender/draw/engines/eevee/eevee_lightcache.c
index 71082268f10..74cac25367c 100644
--- a/source/blender/draw/engines/eevee/eevee_lightcache.c
+++ b/source/blender/draw/engines/eevee/eevee_lightcache.c
@@ -411,7 +411,8 @@ static void eevee_lightbake_count_probes(EEVEE_LightBake *lbake)
 
 	DEG_OBJECT_ITER_FOR_RENDER_ENGINE_BEGIN(depsgraph, ob)
 	{
-		if (!BKE_object_is_visible(ob, OB_VISIBILITY_CHECK_FOR_RENDER)) {
+		const int ob_visibility = BKE_object_visibility(ob, DAG_EVAL_RENDER);
+		if ((ob_visibility & OB_VISIBLE_SELF) == 0) {
 			continue;
 		}
 
@@ -1006,7 +1007,8 @@ static void eevee_lightbake_gather_probes(EEVEE_LightBake *lbake)
 	 * This allows a large number of probe to be precomputed (even dupli ones). */
 	DEG_OBJECT_ITER_FOR_RENDER_ENGINE_BEGIN(depsgraph, ob)
 	{
-		if (!BKE_object_is_visible(ob, OB_VISIBILITY_CHECK_FOR_RENDER)) {
+		const int ob_visibility = BKE_object_visibility(ob, DAG_EVAL_RENDER);
+		if ((ob_visibility & OB_VISIBLE_SELF) == 0) {
 			continue;
 		}
 
diff --git a/source/blender/draw/engines/eevee/eevee_render.c b/source/blender/draw/engines/eevee/eevee_render.c
index 8d196ee07eb..45bfba09890 100644
--- a/source/blender/draw/engines/eevee/eevee_render.c
+++ b/source/blender/draw/engines/eevee/eevee_render.c
@@ -34,6 +34,7 @@
 #include "DNA_object_types.h"
 
 #include "BKE_camera.h"
+#include "BKE_object.h"
 
 #include "BLI_rand.h"
 #include "BLI_rect.h"
@@ -179,11 +180,12 @@ void EEVEE_render_cache(
 		RE_engine_update_stats(engine, NULL, info);
 	}
 
-	if (ob->base_flag & BASE_VISIBLE) {
+	const int ob_visibility = DRW_object_visibility_in_active_context(ob);
+	if (ob_visibility & OB_VISIBLE_PARTICLES) {
 		EEVEE_hair_cache_populate(vedata, sldata, ob, &cast_shadow);
 	}
 
-	if (DRW_object_is_visible_in_active_context(ob)) {
+	if (ob_visibility & OB_VISIBLE_SELF) {
 		if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {
 			EEVEE_materials_cache_populate(vedata, sldata, ob, &cast_shadow);
 		}
diff --git a/source/blender/draw/engines/gpencil/gpencil_engine.c b/source/blender/draw/engines/gpencil/gpencil_engine.c
index 23c53977445..79724f6a5d2 100644
--- a/source/blender/draw/engines/gpencil/gpencil_engine.c
+++ b/source/blender/draw/engines/gpencil/gpencil_engine.c
@@ -559,7 +559,7 @@ static void gpencil_add_draw_data(void *vedata, Object *ob)
 void GPENCIL_cache_populate(void *vedata, Object *ob)
 {
 	/* object must be visible */
-	if (!DRW_object_is_visible_in_active_context(ob)) {
+	if (!(DRW_object_visibility_in_active_context(ob) & OB_VISIBLE_SELF)) {
 		return;
 	}
 
diff --git a/source/blender/draw/engines/gpencil/gpencil_render.c b/source/blender/draw/engines/gpencil/gpencil_render.c
index 51a8209fbf2..1fe008e4d44 100644
--- a/source/blender/draw/engines/gpencil/gpencil_render.c
+++ b/source/blender/draw/engines/gpencil/gpencil_render.c
@@ -28,6 +28,7 @@
 #include "DRW_render.h"
 
 #include "BKE_camera.h"
+#include "BKE_object.h"
 
 #include "DNA_gpencil_types.h"
 
@@ -130,12 +131,10 @@ static void GPENCIL_render_cache(
 	void *vedata, struct Object *ob,
 	struct RenderEngine *UNUSED(engine), struct Depsgraph *UNUSED(depsgraph))
 {
-	if ((ob == NULL) || (DRW_object_is_visible_in_active_context(ob) == false)) {
-		return;
-	}
-
-	if (ob->type == OB_GPENCIL) {
-		GPENCIL_cache_populate(vedata, ob);
+	if (ob && ob->type == OB_GPENCIL) {
+		if (DRW_object_visibility_in_active_context(ob) & OB_VISIBLE_SELF) {
+			GPENCIL_cache_populate(vedata, ob);
+		}
 	}
 }
 
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index c3b469674d3..52caeb6cbf4 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -35,6 +35,7 @@
 
 #include "BKE_node.h"
 #include "BKE_modifier.h"
+#include "BKE_object.h"
 #include "BKE_particle.h"
 
 #include "DNA_image_types.h"
@@ -808,7 +809,10 @@ void workbench_deferred_solid_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 		return; /* Do not draw solid in this case. */
 	}
 
-	if (!DRW_object_is_visible_in_active_context(ob) || (ob->dt < OB_SOLID)) {
+	if (!(DRW_object_visibility_in_active_context(ob) & OB_VISIBLE_SELF)) {
+		return;
+	}
+	if (ob->dt < OB_SOLID) {
 		return;
 	}
 
diff --git a/source/blender/draw/engines/workbench/workbench_forward.c b/source/blender/draw/engines/workbench/workbench_forward.c
index ca02cc2b5ea..6c955ac1fcb 100644
--- a/source/blender/draw/engines/workbench/workbench_forward.c
+++ b/source/blender/draw/engines/workbench/workbench_forward.c
@@ -34,6 +34,7 @@
 #include "BKE_node.h"
 #include "BKE_particle.h"
 #include "BKE_modifier.h"
+#include "BKE_object.h"
 
 #include "DNA_image_types.h"
 #include "DNA_mesh_types.h"
@@ -489,7 +490,10 @@ void workbench_forward_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 		return; /* Do not draw solid in this case. */
 	}
 
-	if (!DRW_object_is_visible_in_active_context(ob) || (ob->dt < OB_WIRE)) {
+	if (!(DRW_object_visibility_in_active_context(ob) & OB_VISIBLE_SELF)) {
+		return;
+	}
+	if (ob->dt < OB_WIRE) {
 		return;
 	}