USD Import: support instance collections.

Added an experimental Instancing option to support
importing USD scenegraph instances as Blender
instanced collections.

9 files changed, 370 insertions, 6 deletions

diff --git a/source/blender/editors/io/io_usd.c b/source/blender/editors/io/io_usd.c
index 068def29f70..a1f2d14b606 100644
--- a/source/blender/editors/io/io_usd.c
+++ b/source/blender/editors/io/io_usd.c
@@ -311,6 +311,8 @@ static int wm_usd_import_exec(bContext *C, wmOperator *op)
   const bool import_proxy = RNA_boolean_get(op->ptr, "import_proxy");
   const bool import_render = RNA_boolean_get(op->ptr, "import_render");
 
+  const bool use_instancing = RNA_boolean_get(op->ptr, "use_instancing");
+
   int offset = 0;
   int sequence_len = 1;
 
@@ -352,6 +354,7 @@ static int wm_usd_import_exec(bContext *C, wmOperator *op)
       import_proxy,
       import_render,
       import_visible_only,
+      use_instancing,
   };
 
   bool ok = USD_import(C, filename, &params, as_background_job);
@@ -437,6 +440,9 @@ static void wm_usd_import_draw(bContext *UNUSED(C), wmOperator *op)
   uiItemR(box, ptr, "import_proxy", 0, NULL, ICON_NONE);
   uiItemR(box, ptr, "import_render", 0, NULL, ICON_NONE);
 
+  box = uiLayoutBox(layout);
+  uiItemL(box, IFACE_("Experimental"), ICON_NONE);
+  uiItemR(box, ptr, "use_instancing", 0, NULL, ICON_NONE);
 }
 
 void WM_OT_usd_import(struct wmOperatorType *ot)
@@ -539,7 +545,8 @@ void WM_OT_usd_import(struct wmOperatorType *ot)
                   true,
                   "Import Instance Proxies",
                   "If enabled, USD instances will be traversed with instance proxies, "
-                  "creating a unique Blender object for each instance");
+                  "creating a unique Blender object for each instance.  Note that "
+                  "this option is ignored if the Instancing option is also checked"  );
 
   RNA_def_boolean(ot->srna,
                   "import_visible_only",
@@ -578,6 +585,14 @@ void WM_OT_usd_import(struct wmOperatorType *ot)
 
   RNA_def_boolean(
     ot->srna, "import_render", true, "Render", "When checked, import final render geometry");
+
+  RNA_def_boolean(
+    ot->srna,
+    "use_instancing",
+    false,
+    "Instancing",
+    "When checked, USD scenegraph instances are imported as collection instances in Blender. "
+    "(Note that point instancers are not yet handled by this option.)");
 }
 
 #endif /* WITH_USD */
diff --git a/source/blender/io/usd/CMakeLists.txt b/source/blender/io/usd/CMakeLists.txt
index 46c2ac26e94..70a1045be85 100644
--- a/source/blender/io/usd/CMakeLists.txt
+++ b/source/blender/io/usd/CMakeLists.txt
@@ -68,6 +68,7 @@ set(SRC
   intern/usd_reader_camera.cc
   intern/usd_reader_curve.cc
   intern/usd_reader_geom.cc
+  intern/usd_reader_instance.cc
   intern/usd_reader_light.cc
   intern/usd_reader_mesh.cc
   intern/usd_reader_nurbs.cc
@@ -92,6 +93,7 @@ set(SRC
   intern/usd_reader_camera.h
   intern/usd_reader_curve.h
   intern/usd_reader_geom.h
+  intern/usd_reader_instance.h
   intern/usd_reader_light.h
   intern/usd_reader_mesh.h
   intern/usd_reader_nurbs.h
diff --git a/source/blender/io/usd/intern/usd_capi.cc b/source/blender/io/usd/intern/usd_capi.cc
index e808cba4c64..dd4ca6e28da 100644
--- a/source/blender/io/usd/intern/usd_capi.cc
+++ b/source/blender/io/usd/intern/usd_capi.cc
@@ -19,6 +19,7 @@
 
 #include "usd.h"
 #include "usd_hierarchy_iterator.h"
+#include "usd_reader_instance.h"
 #include "usd_reader_mesh.h"
 #include "usd_reader_prim.h"
 #include "usd_reader_stage.h"
@@ -74,6 +75,8 @@
 #include "usd_reader_prim.h"
 #include "usd_util.h"
 
+#include <iostream>
+
 struct USDStageHandle {
   CacheArchiveHandle base;
 };
@@ -114,6 +117,128 @@ static bool gather_objects_paths(const pxr::UsdPrim &object, ListBase *object_pa
   return true;
 }
 
+/* Create a collection with the given parent and name. */
+static Collection *create_collection(Main *bmain, Collection *parent, const char *name)
+{
+  if (!bmain) {
+    return nullptr;
+  }
+
+  Collection *coll = BKE_collection_add(bmain, parent, name);
+
+  if (coll) {
+    id_fake_user_set(&coll->id);
+    DEG_id_tag_update(&coll->id, ID_RECALC_COPY_ON_WRITE);
+  }
+
+  return coll;
+}
+
+/* Set the instance collection on the given instance reader.
+*  The collection is assigned from the given map based on
+*  the prototype (maser) prim path. */
+static void set_instance_collection(
+  USDInstanceReader *instance_reader,
+  const std::map<pxr::SdfPath, Collection *> &proto_collection_map)
+{
+  if (!instance_reader) {
+    return;
+  }
+
+  pxr::SdfPath proto_path = instance_reader->proto_path();
+
+  std::map<pxr::SdfPath, Collection *>::const_iterator it = proto_collection_map.find(proto_path);
+
+  if (it != proto_collection_map.end()) {
+    instance_reader->set_instance_collection(it->second);
+  }
+  else {
+    std::cerr << "WARNING: Couldn't find prototype collection for " << instance_reader->prim_path()
+      << std::endl;
+  }
+}
+
+/* Create instance collections for the USD instance readers. */
+static void create_proto_collections(
+  Main *bmain,
+  ViewLayer *view_layer,
+  Collection *parent_collection,
+  const USDStageReader::ProtoReaderMap &proto_readers,
+  const std::vector<USDPrimReader *> &readers)
+{
+  Collection *all_protos_collection = create_collection(bmain, parent_collection, "prototypes");
+
+  std::map<pxr::SdfPath, Collection *> proto_collection_map;
+
+  for (const auto &pair : proto_readers) {
+
+    std::string proto_collection_name = pair.first.GetString();
+
+    // TODO(makowalski): Is it acceptable to have slashes in the collection names? Or should we
+    // replace them with another character, like an underscore, as in the following?
+    // std::replace(proto_collection_name.begin(), proto_collection_name.end(), '/', '_');
+
+    Collection *proto_collection = create_collection(
+      bmain, all_protos_collection, proto_collection_name.c_str());
+
+    LayerCollection *proto_lc = BKE_layer_collection_first_from_scene_collection(view_layer,
+      proto_collection);
+    if (proto_lc) {
+      proto_lc->flag |= LAYER_COLLECTION_HIDE;
+    }
+
+    proto_collection_map.insert(std::make_pair(pair.first, proto_collection));
+  }
+
+  // Set the instance collections on the readers, including the prototype
+  // readers, as instancing may be recursive.
+
+  for (const auto &pair : proto_readers) {
+    for (USDPrimReader *reader : pair.second) {
+      if (USDInstanceReader *instance_reader = dynamic_cast<USDInstanceReader *>(reader)) {
+        set_instance_collection(instance_reader, proto_collection_map);
+      }
+    }
+  }
+
+  for (USDPrimReader *reader : readers) {
+    if (USDInstanceReader *instance_reader = dynamic_cast<USDInstanceReader *>(reader)) {
+      set_instance_collection(instance_reader, proto_collection_map);
+    }
+  }
+
+  // Add the prototype objects to the collections.
+  for (const auto &pair : proto_readers) {
+
+    std::map<pxr::SdfPath, Collection *>::const_iterator it = proto_collection_map.find(
+      pair.first);
+
+    if (it == proto_collection_map.end()) {
+      std::cerr << "WARNING: Couldn't find collection when adding objects for prototype "
+        << pair.first << std::endl;
+      continue;
+    }
+
+    for (USDPrimReader *reader : pair.second) {
+      Object *ob = reader->object();
+
+      if (!ob) {
+        continue;
+      }
+
+      Collection *coll = it->second;
+
+      BKE_collection_object_add(bmain, coll, ob);
+
+      DEG_id_tag_update(&coll->id, ID_RECALC_COPY_ON_WRITE);
+      DEG_id_tag_update_ex(bmain,
+        &ob->id,
+        ID_RECALC_TRANSFORM | ID_RECALC_GEOMETRY | ID_RECALC_ANIMATION |
+        ID_RECALC_BASE_FLAGS);
+    }
+  }
+}
+
 /* ********************** Export file ********************** */
 
 struct ExportJobData {
@@ -444,9 +569,38 @@ static void import_startjob(void *customdata, short *stop, short *do_update, flo
   const float size = static_cast<float>(data->readers.size());
   size_t i = 0;
 
+  /* Setup parenthood */
+
+  /* Handle instance prototypes.
+  /* TODO(makowalski): Move this logic inside USDReaderStage? */
+  for (const auto &pair : archive->proto_readers()) {
+
+    for (USDPrimReader *reader : pair.second) {
+
+      if (!reader) {
+        continue;
+      }
+
+      /* TODO(makowalski): Here and below, should we call
+      *  readObjectData() with the actual time? */
+      reader->readObjectData(data->bmain, 0.0);
+
+      Object *ob = reader->object();
+
+      if (!ob) {
+        continue;
+      }
+
+      const USDPrimReader *parent_reader = reader->parent();
+
+      ob->parent = parent_reader ? parent_reader->object() : nullptr;
+
+      // TODO(makowalski): Handle progress update.
+    }
+  }
+
   std::vector<USDPrimReader *>::iterator iter;
 
-  /* Setup parenthood */
   for (iter = data->readers.begin(); iter != data->readers.end(); ++iter) {
     Object *ob = (*iter)->object();
 
@@ -498,6 +652,19 @@ static void import_endjob(void *customdata)
 
       BKE_id_free_us(data->bmain, ob);
     }
+
+    if (data->archive) {
+      for (const auto &pair : data->archive->proto_readers()) {
+        for (USDPrimReader *reader : pair.second) {
+          Object *ob = reader->object();
+          /* It's possible that cancellation occurred between the creation of
+           * the reader and the creation of the Blender object. */
+          if (ob != NULL) {
+            BKE_id_free_us(data->bmain, ob);
+          }
+        }
+      }
+    }
   }
   else {
     /* Add object to scene. */
@@ -509,6 +676,11 @@ static void import_endjob(void *customdata)
 
     lc = BKE_layer_collection_get_active(view_layer);
 
+    if (data->archive && !data->archive->proto_readers().empty()) {
+      create_proto_collections(
+        data->bmain, view_layer, lc->collection, data->archive->proto_readers(), data->readers);
+    }
+
     for (iter = data->readers.begin(); iter != data->readers.end(); ++iter) {
       Object *ob = (*iter)->object();
 
@@ -538,6 +710,8 @@ static void import_endjob(void *customdata)
     }
   }
 
+  data->archive->clear_proto_readers(true);
+
   WM_set_locked_interface(data->wm, false);
 
   switch (data->error_code) {
diff --git a/source/blender/io/usd/intern/usd_reader_instance.cc b/source/blender/io/usd/intern/usd_reader_instance.cc
new file mode 100644
index 00000000000..bb0ac3c1e2b
--- /dev/null
+++ b/source/blender/io/usd/intern/usd_reader_instance.cc
@@ -0,0 +1,65 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation.
+ * All rights reserved.
+ */
+
+#include "usd_reader_instance.h"
+
+#include "BKE_object.h"
+#include "DNA_object_types.h"
+
+#include <iostream>
+
+namespace blender::io::usd {
+
+ USDInstanceReader::USDInstanceReader(pxr::UsdStageRefPtr stage,
+                                      const pxr::UsdPrim &object,
+                                      const USDImportParams &import_params,
+                                      ImportSettings &settings)
+   : USDXformReader(stage, object, import_params, settings)
+ {
+ }
+
+ bool USDInstanceReader::valid() const
+ {
+   return m_prim.IsValid() && m_prim.IsInstance();
+ }
+
+ void USDInstanceReader::createObject(Main *bmain, double motionSampleTime)
+ {
+   this->m_object = BKE_object_add_only_object(bmain, OB_EMPTY, m_name.c_str());
+   this->m_object->data = nullptr;
+   this->m_object->transflag |= OB_DUPLICOLLECTION;
+ }
+
+ void USDInstanceReader::set_instance_collection(Collection *coll)
+ {
+   if (coll && this->m_object) {
+     this->m_object->instance_collection = coll;
+   }
+ }
+
+ pxr::SdfPath USDInstanceReader::proto_path() const
+ {
+   if (pxr::UsdPrim master = m_prim.GetMaster()) {
+     return master.GetPath();
+   }
+
+   return pxr::SdfPath();
+ }
+
+}  // namespace blender::io::usd
diff --git a/source/blender/io/usd/intern/usd_reader_instance.h b/source/blender/io/usd/intern/usd_reader_instance.h
new file mode 100644
index 00000000000..ce5c7ce6a83
--- /dev/null
+++ b/source/blender/io/usd/intern/usd_reader_instance.h
@@ -0,0 +1,49 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2021 Blender Foundation.
+ * All rights reserved.
+ */
+#pragma once
+
+#include "usd_reader_xform.h"
+
+#include <pxr/usd/usdGeom/xform.h>
+
+struct Collection;
+
+namespace blender::io::usd {
+
+/* Wraps the UsdGeomXform schema. Creates a Blender Empty object. */
+
+ class USDInstanceReader : public USDXformReader {
+
+  public:
+   USDInstanceReader(pxr::UsdStageRefPtr stage,
+                     const pxr::UsdPrim &object,
+                     const USDImportParams &import_params,
+                     ImportSettings &settings);
+
+   bool valid() const override;
+
+   void createObject(Main *bmain, double motionSampleTime) override;
+
+   void set_instance_collection(Collection *coll);
+
+   pxr::SdfPath proto_path() const;
+
+ };
+
+}  // namespace blender::io::usd
diff --git a/source/blender/io/usd/intern/usd_reader_stage.cc b/source/blender/io/usd/intern/usd_reader_stage.cc
index 8c6023d4805..e27862e4cf3 100644
--- a/source/blender/io/usd/intern/usd_reader_stage.cc
+++ b/source/blender/io/usd/intern/usd_reader_stage.cc
@@ -139,8 +139,10 @@ static USDPrimReader *_handlePrim(Main *bmain,
 
   USDPrimReader *reader = NULL;
 
-  // This check prevents the pseudo 'root' prim to be added
-  if (prim != stage->GetPseudoRoot()) {
+  // This check prevents the stage pseudo 'root' prim
+  // or the root prims of scenegraph 'master' prototypes
+  // from being added.
+  if (!(prim.IsPseudoRoot() || prim.IsMaster())) {
     reader = blender::io::usd::create_reader(stage, prim, params, settings);
     if (reader == NULL)
       return NULL;
@@ -154,7 +156,7 @@ static USDPrimReader *_handlePrim(Main *bmain,
 
   pxr::Usd_PrimFlagsPredicate filter_predicate = pxr::UsdPrimDefaultPredicate;
 
-  if (params.import_instance_proxies) {
+  if (!params.use_instancing && params.import_instance_proxies) {
     filter_predicate = pxr::UsdTraverseInstanceProxies(filter_predicate);
   }
 
@@ -201,6 +203,17 @@ std::vector<USDPrimReader *> USDStageReader::collect_readers(Main *bmain,
   m_stage->SetInterpolationType(pxr::UsdInterpolationType::UsdInterpolationTypeHeld);
   _handlePrim(bmain, m_stage, params, root, NULL, m_readers, settings);
 
+  if (params.use_instancing) {
+    // Collect the scenegraph instance prototypes.
+    std::vector<pxr::UsdPrim> protos = m_stage->GetMasters();
+
+    for (const pxr::UsdPrim &proto_prim : protos) {
+      std::vector<USDPrimReader *> proto_readers;
+      _handlePrim(bmain, m_stage, params, proto_prim, NULL, proto_readers, settings);
+      m_proto_readers.insert(std::make_pair(proto_prim.GetPath(), proto_readers));
+    }
+  }
+
   return m_readers;
 }
 
@@ -212,6 +225,33 @@ void USDStageReader::clear_readers()
       delete *iter;
     }
   }
+
+  m_readers.clear();
+}
+
+void USDStageReader::clear_proto_readers(bool decref)
+{
+  for (auto &pair : m_proto_readers) {
+
+    for (USDPrimReader *reader : pair.second) {
+
+      if (!reader) {
+        continue;
+      }
+
+      if (decref) {
+        reader->decref();
+      }
+
+      if (reader->refcount() == 0) {
+        delete reader;
+      }
+    }
+
+    pair.second.clear();
+  }
+
+  m_proto_readers.clear();
 }
 
 }  // Namespace blender::io::usd
diff --git a/source/blender/io/usd/intern/usd_reader_stage.h b/source/blender/io/usd/intern/usd_reader_stage.h
index 0bd9a40d5ad..a9502cee574 100644
--- a/source/blender/io/usd/intern/usd_reader_stage.h
+++ b/source/blender/io/usd/intern/usd_reader_stage.h
@@ -33,6 +33,7 @@ struct Scene;
 #include <pxr/pxr.h>
 #include <pxr/usd/usd/stage.h>
 
+#include <map>
 #include <vector>
 
 struct ImportSettings;
@@ -45,12 +46,19 @@ namespace blender::io::usd {
  */
 
 class USDStageReader {
+ public:
+  typedef std::map<pxr::SdfPath, std::vector<USDPrimReader *>> ProtoReaderMap;
+
+ protected:
   pxr::UsdStageRefPtr m_stage;
   USDImportParams m_params;
   ImportSettings m_settings;
 
   std::vector<USDPrimReader *> m_readers;
 
+  // Readers for scenegraph instance prototypes.
+  ProtoReaderMap m_proto_readers;
+
  public:
   USDStageReader(struct Main *bmain, const char *filename);
   ~USDStageReader();
@@ -84,6 +92,10 @@ class USDStageReader {
   }
 
   void clear_readers();
+
+  void clear_proto_readers(bool decref);
+
+  const ProtoReaderMap &proto_readers() const { return m_proto_readers; };
 };
 
 };  // namespace blender::io::usd
diff --git a/source/blender/io/usd/intern/usd_util.cc b/source/blender/io/usd/intern/usd_util.cc
index 4a218b954f0..80ff20c513a 100644
--- a/source/blender/io/usd/intern/usd_util.cc
+++ b/source/blender/io/usd/intern/usd_util.cc
@@ -22,6 +22,7 @@
 #include "usd_reader_camera.h"
 #include "usd_reader_curve.h"
 #include "usd_reader_geom.h"
+#include "usd_reader_instance.h"
 #include "usd_reader_light.h"
 #include "usd_reader_mesh.h"
 #include "usd_reader_nurbs.h"
@@ -93,7 +94,10 @@ USDPrimReader *create_reader(const pxr::UsdStageRefPtr &stage,
 {
   USDPrimReader *reader = nullptr;
 
-  if (params.import_cameras && prim.IsA<pxr::UsdGeomCamera>()) {
+  if (params.use_instancing && prim.IsInstance()) {
+    reader = new USDInstanceReader(stage, prim, params, settings);
+  }
+  else if (params.import_cameras && prim.IsA<pxr::UsdGeomCamera>()) {
     reader = new USDCameraReader(stage, prim, params, settings);
   }
   else if (params.import_curves && prim.IsA<pxr::UsdGeomBasisCurves>()) {
@@ -122,6 +126,8 @@ USDPrimReader *create_reader(const pxr::UsdStageRefPtr &stage,
 USDPrimReader *create_fake_reader(USDStageReader *archive, const pxr::UsdPrim &prim)
 {
   USDPrimReader *reader = nullptr;
+
+  // TODO(makowalski): Handle true instancing?
   if (prim.IsA<pxr::UsdGeomCamera>()) {
     reader = new USDCameraReader(archive->stage(), prim, archive->params(), archive->settings());
   }
diff --git a/source/blender/io/usd/usd.h b/source/blender/io/usd/usd.h
index edf55f3700d..e97a099b0b2 100644
--- a/source/blender/io/usd/usd.h
+++ b/source/blender/io/usd/usd.h
@@ -67,6 +67,7 @@ struct USDImportParams {
   bool import_proxy;
   bool import_render;
   bool import_visible_only;
+  bool use_instancing;
 };
 
 /* The USD_export takes a as_background_job parameter, and returns a boolean.