Fix T75936: Alembic, allow exporting of invisible objects

Add a new depsgraph builder class that includes invisible objects and
use that in the Alembic exporter.

Alembic supports three options for visibility, "visible", "inherited",
and "hidden". This means that parents can be hidden and still have
visible children (contrary to USD, where invisibility is used to prune
an entire scene graph subtree). Because of this, the visibility is
stored on the transform node, as that represents the Object in Blender
and thus keeps the Alembic file as close to Blender's own structure as
possible.

Reviewed By: Sergey

Differential Revision: https://developer.blender.org/D8595

12 files changed, 239 insertions, 17 deletions

diff --git a/source/blender/depsgraph/CMakeLists.txt b/source/blender/depsgraph/CMakeLists.txt
index f9a6bde442d..e0916491edb 100644
--- a/source/blender/depsgraph/CMakeLists.txt
+++ b/source/blender/depsgraph/CMakeLists.txt
@@ -55,6 +55,7 @@ set(SRC
   intern/builder/deg_builder_rna.cc
   intern/builder/deg_builder_transitive.cc
   intern/builder/pipeline.cc
+  intern/builder/pipeline_all_objects.cc
   intern/builder/pipeline_compositor.cc
   intern/builder/pipeline_from_ids.cc
   intern/builder/pipeline_render.cc
@@ -116,6 +117,7 @@ set(SRC
   intern/builder/deg_builder_rna.h
   intern/builder/deg_builder_transitive.h
   intern/builder/pipeline.h
+  intern/builder/pipeline_all_objects.h
   intern/builder/pipeline_compositor.h
   intern/builder/pipeline_from_ids.h
   intern/builder/pipeline_render.h
diff --git a/source/blender/depsgraph/DEG_depsgraph_build.h b/source/blender/depsgraph/DEG_depsgraph_build.h
index 50f22b00028..dd52c97e03f 100644
--- a/source/blender/depsgraph/DEG_depsgraph_build.h
+++ b/source/blender/depsgraph/DEG_depsgraph_build.h
@@ -56,6 +56,12 @@ void DEG_graph_build_from_view_layer(struct Depsgraph *graph,
                                      struct Scene *scene,
                                      struct ViewLayer *view_layer);
 
+/* Build depsgraph for all objects (so also invisible ones) in the given view layer. */
+void DEG_graph_build_for_all_objects(struct Depsgraph *graph,
+                                     struct Main *bmain,
+                                     struct Scene *scene,
+                                     struct ViewLayer *view_layer);
+
 /* Special version of builder which produces dependency graph suitable for the render pipeline.
  * It will contain sequencer and compositor (if needed) and all their dependencies. */
 void DEG_graph_build_for_render_pipeline(struct Depsgraph *graph,
diff --git a/source/blender/depsgraph/intern/builder/pipeline_all_objects.cc b/source/blender/depsgraph/intern/builder/pipeline_all_objects.cc
new file mode 100644
index 00000000000..c926ff7541a
--- /dev/null
+++ b/source/blender/depsgraph/intern/builder/pipeline_all_objects.cc
@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2020 Blender Foundation.
+ * All rights reserved.
+ */
+
+#include "pipeline_all_objects.h"
+
+#include "intern/builder/deg_builder_nodes.h"
+#include "intern/builder/deg_builder_relations.h"
+#include "intern/depsgraph.h"
+
+#include "DNA_layer_types.h"
+
+namespace blender {
+namespace deg {
+
+namespace {
+
+class AllObjectsNodeBuilder : public DepsgraphNodeBuilder {
+ public:
+  AllObjectsNodeBuilder(Main *bmain, Depsgraph *graph, DepsgraphBuilderCache *cache)
+      : DepsgraphNodeBuilder(bmain, graph, cache)
+  {
+  }
+
+  virtual bool need_pull_base_into_graph(Base * /*base*/) override
+  {
+    return true;
+  }
+};
+
+class AllObjectsRelationBuilder : public DepsgraphRelationBuilder {
+ public:
+  AllObjectsRelationBuilder(Main *bmain, Depsgraph *graph, DepsgraphBuilderCache *cache)
+      : DepsgraphRelationBuilder(bmain, graph, cache)
+  {
+  }
+
+  virtual bool need_pull_base_into_graph(Base * /*base*/) override
+  {
+    return true;
+  }
+};
+
+}  // namespace
+
+AllObjectsBuilderPipeline::AllObjectsBuilderPipeline(::Depsgraph *graph,
+                                                     Main *bmain,
+                                                     Scene *scene,
+                                                     ViewLayer *view_layer)
+    : ViewLayerBuilderPipeline(graph, bmain, scene, view_layer)
+{
+}
+
+unique_ptr<DepsgraphNodeBuilder> AllObjectsBuilderPipeline::construct_node_builder()
+{
+  return std::make_unique<AllObjectsNodeBuilder>(bmain_, deg_graph_, &builder_cache_);
+}
+
+unique_ptr<DepsgraphRelationBuilder> AllObjectsBuilderPipeline::construct_relation_builder()
+{
+  return std::make_unique<AllObjectsRelationBuilder>(bmain_, deg_graph_, &builder_cache_);
+}
+
+}  // namespace deg
+}  // namespace blender
diff --git a/source/blender/depsgraph/intern/builder/pipeline_all_objects.h b/source/blender/depsgraph/intern/builder/pipeline_all_objects.h
new file mode 100644
index 00000000000..94f00ae840b
--- /dev/null
+++ b/source/blender/depsgraph/intern/builder/pipeline_all_objects.h
@@ -0,0 +1,44 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2020 Blender Foundation.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup depsgraph
+ */
+
+#pragma once
+
+#include "pipeline_view_layer.h"
+
+namespace blender {
+namespace deg {
+
+/* Builds a dependency graph that contains all objects in the view layer.
+ * This is contrary to the regular ViewLayerBuilderPipeline, which is limited to visible objects
+ * (and their dependencies). */
+class AllObjectsBuilderPipeline : public ViewLayerBuilderPipeline {
+ public:
+  AllObjectsBuilderPipeline(::Depsgraph *graph, Main *bmain, Scene *scene, ViewLayer *view_layer);
+
+ protected:
+  virtual unique_ptr<DepsgraphNodeBuilder> construct_node_builder() override;
+  virtual unique_ptr<DepsgraphRelationBuilder> construct_relation_builder() override;
+};
+
+}  // namespace deg
+}  // namespace blender
diff --git a/source/blender/depsgraph/intern/depsgraph_build.cc b/source/blender/depsgraph/intern/depsgraph_build.cc
index fb933cb38f3..230c59bd651 100644
--- a/source/blender/depsgraph/intern/depsgraph_build.cc
+++ b/source/blender/depsgraph/intern/depsgraph_build.cc
@@ -44,6 +44,7 @@
 #include "DEG_depsgraph_debug.h"
 
 #include "builder/deg_builder_relations.h"
+#include "builder/pipeline_all_objects.h"
 #include "builder/pipeline_compositor.h"
 #include "builder/pipeline_from_ids.h"
 #include "builder/pipeline_render.h"
@@ -218,6 +219,15 @@ void DEG_graph_build_from_view_layer(Depsgraph *graph,
   builder.build();
 }
 
+void DEG_graph_build_for_all_objects(struct Depsgraph *graph,
+                                     struct Main *bmain,
+                                     struct Scene *scene,
+                                     struct ViewLayer *view_layer)
+{
+  deg::AllObjectsBuilderPipeline builder(graph, bmain, scene, view_layer);
+  builder.build();
+}
+
 void DEG_graph_build_for_render_pipeline(Depsgraph *graph,
                                          Main *bmain,
                                          Scene *scene,
diff --git a/source/blender/io/alembic/exporter/abc_export_capi.cc b/source/blender/io/alembic/exporter/abc_export_capi.cc
index 8c5f3d89870..c4966a965eb 100644
--- a/source/blender/io/alembic/exporter/abc_export_capi.cc
+++ b/source/blender/io/alembic/exporter/abc_export_capi.cc
@@ -67,11 +67,17 @@ namespace io {
 namespace alembic {
 
 // Construct the depsgraph for exporting.
-static void build_depsgraph(Depsgraph *depsgraph, Main *bmain)
+static void build_depsgraph(Depsgraph *depsgraph, Main *bmain, const bool visible_objects_only)
 {
   Scene *scene = DEG_get_input_scene(depsgraph);
   ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
-  DEG_graph_build_from_view_layer(depsgraph, bmain, scene, view_layer);
+
+  if (visible_objects_only) {
+    DEG_graph_build_from_view_layer(depsgraph, bmain, scene, view_layer);
+  }
+  else {
+    DEG_graph_build_for_all_objects(depsgraph, bmain, scene, view_layer);
+  }
 }
 
 static void export_startjob(void *customdata,
@@ -91,7 +97,7 @@ static void export_startjob(void *customdata,
   *progress = 0.0f;
   *do_update = true;
 
-  build_depsgraph(data->depsgraph, data->bmain);
+  build_depsgraph(data->depsgraph, data->bmain, data->params.visible_objects_only);
   SubdivModifierDisabler subdiv_disabler(data->depsgraph);
   if (!data->params.apply_subdiv) {
     subdiv_disabler.disable_modifiers();
diff --git a/source/blender/io/alembic/exporter/abc_writer_abstract.cc b/source/blender/io/alembic/exporter/abc_writer_abstract.cc
index e43b394e27f..84527a12e85 100644
--- a/source/blender/io/alembic/exporter/abc_writer_abstract.cc
+++ b/source/blender/io/alembic/exporter/abc_writer_abstract.cc
@@ -25,6 +25,10 @@
 
 #include "DNA_modifier_types.h"
 
+#include "DEG_depsgraph.h"
+
+#include <Alembic/AbcGeom/Visibility.h>
+
 #include "CLG_log.h"
 static CLG_LogRef LOG = {"io.alembic"};
 
@@ -96,6 +100,18 @@ void ABCAbstractWriter::update_bounding_box(Object *object)
   bounding_box_.max.z = -bb->vec[0][1];
 }
 
+void ABCAbstractWriter::write_visibility(const HierarchyContext &context)
+{
+  const bool is_visible = context.is_object_visible(DAG_EVAL_RENDER);
+  Alembic::Abc::OObject abc_object = get_alembic_object();
+
+  if (!abc_visibility_.valid()) {
+    abc_visibility_ = Alembic::AbcGeom::CreateVisibilityProperty(abc_object, timesample_index_);
+  }
+  abc_visibility_.set(is_visible ? Alembic::AbcGeom::kVisibilityVisible :
+                                   Alembic::AbcGeom::kVisibilityHidden);
+}
+
 }  // namespace alembic
 }  // namespace io
 }  // namespace blender
diff --git a/source/blender/io/alembic/exporter/abc_writer_abstract.h b/source/blender/io/alembic/exporter/abc_writer_abstract.h
index a83373a567a..f46409b7902 100644
--- a/source/blender/io/alembic/exporter/abc_writer_abstract.h
+++ b/source/blender/io/alembic/exporter/abc_writer_abstract.h
@@ -43,6 +43,9 @@ class ABCAbstractWriter : public AbstractHierarchyWriter {
   uint32_t timesample_index_;
   Imath::Box3d bounding_box_;
 
+  /* Visibility of this writer's data in Alembic. */
+  Alembic::Abc::OCharProperty abc_visibility_;
+
  public:
   explicit ABCAbstractWriter(const ABCWriterConstructorArgs &args);
   virtual ~ABCAbstractWriter();
@@ -70,6 +73,8 @@ class ABCAbstractWriter : public AbstractHierarchyWriter {
   virtual void do_write(HierarchyContext &context) = 0;
 
   virtual void update_bounding_box(Object *object);
+
+  void write_visibility(const HierarchyContext &context);
 };
 
 }  // namespace alembic
diff --git a/source/blender/io/alembic/exporter/abc_writer_mesh.cc b/source/blender/io/alembic/exporter/abc_writer_mesh.cc
index 5c005164bcc..517f0212712 100644
--- a/source/blender/io/alembic/exporter/abc_writer_mesh.cc
+++ b/source/blender/io/alembic/exporter/abc_writer_mesh.cc
@@ -159,7 +159,10 @@ ModifierData *ABCGenericMeshWriter::get_liquid_sim_modifier(Scene *scene, Object
 
 bool ABCGenericMeshWriter::is_supported(const HierarchyContext *context) const
 {
-  return context->is_object_visible(DAG_EVAL_RENDER);
+  if (args_.export_params->visible_objects_only) {
+    return context->is_object_visible(DAG_EVAL_RENDER);
+  }
+  return true;
 }
 
 void ABCGenericMeshWriter::do_write(HierarchyContext &context)
diff --git a/source/blender/io/alembic/exporter/abc_writer_transform.cc b/source/blender/io/alembic/exporter/abc_writer_transform.cc
index 39af99c142c..7694066a13d 100644
--- a/source/blender/io/alembic/exporter/abc_writer_transform.cc
+++ b/source/blender/io/alembic/exporter/abc_writer_transform.cc
@@ -92,6 +92,8 @@ void ABCTransformWriter::do_write(HierarchyContext &context)
   xform_sample.setMatrix(convert_matrix_datatype(parent_relative_matrix));
   xform_sample.setInheritsXforms(true);
   abc_xform_schema_.set(xform_sample);
+
+  write_visibility(context);
 }
 
 const OObject ABCTransformWriter::get_alembic_object() const
diff --git a/source/blender/io/common/intern/abstract_hierarchy_iterator.cc b/source/blender/io/common/intern/abstract_hierarchy_iterator.cc
index 93e7e677118..d825625cafc 100644
--- a/source/blender/io/common/intern/abstract_hierarchy_iterator.cc
+++ b/source/blender/io/common/intern/abstract_hierarchy_iterator.cc
@@ -80,7 +80,7 @@ void HierarchyContext::mark_as_not_instanced()
 
 bool HierarchyContext::is_object_visible(const enum eEvaluationMode evaluation_mode) const
 {
-  bool is_dupli = duplicator != nullptr;
+  const bool is_dupli = duplicator != nullptr;
   int base_flag;
 
   if (is_dupli) {
@@ -92,7 +92,7 @@ bool HierarchyContext::is_object_visible(const enum eEvaluationMode evaluation_m
     object->base_flag = duplicator->base_flag | BASE_FROM_DUPLI;
   }
 
-  int visibility = BKE_object_visibility(object, evaluation_mode);
+  const int visibility = BKE_object_visibility(object, evaluation_mode);
 
   if (is_dupli) {
     object->base_flag = base_flag;
diff --git a/tests/python/alembic_export_tests.py b/tests/python/alembic_export_tests.py
index 800e450776c..cacd2f2dd0e 100644
--- a/tests/python/alembic_export_tests.py
+++ b/tests/python/alembic_export_tests.py
@@ -32,6 +32,7 @@ import pathlib
 import subprocess
 import sys
 import unittest
+from typing import Tuple
 
 from modules.test_utils import (
     with_tempdir,
@@ -59,19 +60,13 @@ class AbstractAlembicTest(AbstractBlenderRunnerTest):
         # 'abcls' array notation, like "name[16]"
         cls.abcls_array = re.compile(r'^(?P<name>[^\[]+)(\[(?P<arraysize>\d+)\])?$')
 
-    def abcprop(self, filepath: pathlib.Path, proppath: str) -> dict:
-        """Uses abcls to obtain compound property values from an Alembic object.
-
-        A dict of subproperties is returned, where the values are Python values.
+    def abcls(self, *arguments) -> Tuple[int, str]:
+        """Uses abcls and return its output.
 
-        The Python bindings for Alembic are old, and only compatible with Python 2.x,
-        so that's why we can't use them here, and have to rely on other tooling.
+        :return: tuple (process exit status code, stdout)
         """
-        import collections
 
-        abcls = self.alembic_root / 'bin' / 'abcls'
-
-        command = (str(abcls), '-vl', '%s%s' % (filepath, proppath))
+        command = (self.alembic_root / 'bin' / 'abcls', *arguments)
         proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                               timeout=30)
 
@@ -84,7 +79,25 @@ class AbstractAlembicTest(AbstractBlenderRunnerTest):
         output = output.replace('\r\n', '\n').replace('\r', '\n')
 
         if proc.returncode:
-            raise AbcPropError('Error %d running %s:\n%s' % (proc.returncode, ' '.join(command), output))
+            str_command = " ".join(str(c) for c in command)
+            print(f'command {str_command} failed with status {proc.returncode}')
+
+        return (proc.returncode, output)
+
+    def abcprop(self, filepath: pathlib.Path, proppath: str) -> dict:
+        """Uses abcls to obtain compound property values from an Alembic object.
+
+        A dict of subproperties is returned, where the values are Python values.
+
+        The Python bindings for Alembic are old, and only compatible with Python 2.x,
+        so that's why we can't use them here, and have to rely on other tooling.
+        """
+        import collections
+
+        command = ('-vl', '%s%s' % (filepath, proppath))
+        returncode, output = self.abcls(*command)
+        if returncode:
+            raise AbcPropError('Error %d running abcls:\n%s' % (returncode, output))
 
         # Mapping from value type to callable that can convert a string to Python values.
         converters = {
@@ -536,6 +549,41 @@ class LongNamesExportTest(AbstractAlembicTest):
         self.assertIn('.faceCounts', abcprop)
 
 
+class InvisibleObjectExportTest(AbstractAlembicTest):
+    """Export an object which is invisible.
+
+    This test only tests a small subset of the functionality that is required to
+    export invisible objects. It just tests that the visibility property is
+    written, and that it has the correct initial value. This is a limitation
+    caused by these tests relying on `abcls`.
+    """
+
+    @with_tempdir
+    def test_hierarchical_export(self, tempdir: pathlib.Path):
+        abc = tempdir / 'visibility.abc'
+        script = "import bpy; bpy.ops.wm.alembic_export(filepath='%s', start=1, end=2, " \
+                 "renderable_only=False, visible_objects_only=False)" % abc.as_posix()
+        self.run_blender('visibility.blend', script)
+
+        def test(cube_name: str, expect_visible: bool):
+            returncode, output = self.abcls('-va', f'{abc}/{cube_name}')
+            if returncode:
+                self.fail(f"abcls failed: {output}")
+            output = output.strip()
+            self.assertEqual(f'Cube   .xform   visible   {int(expect_visible)}', output)
+
+        # This cube is always visible.
+        test('VisibleCube', True)
+
+        # This cube is never visible, and thus will not be pulled into the
+        # depsgraph by the standard builder, only by the all-objects builder.
+        test('InvisibleCube', False)
+
+        # This cube has animated visibility, and thus will be pulled into the
+        # depsgraph by the standard builder as well as the all-objects builder.
+        test('InvisibleAnimatedCube', False)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--blender', required=True)