IO: speed up large Alembic & USD imports by doing fewer collection syncs

Previous code was doing N collection syncs when importing N objects
(essentially quadratic complexity in terms of object count). New
code avoids all the intermediate syncs by using
BKE_layer_collection_resync_forbid and
BKE_layer_collection_resync_allow, and then does one
BKE_main_collection_sync + BKE_main_collection_sync_remap for the
whole operation. The things done on the importer objects that are
dependent on the sync happening (marking them selected) are done in a
separate loop after the sync.

Timings: importing Moana USD scene (480k objects) on Windows, VS2022
Release build, AMD Ryzen 5950X: 12344sec -> 10979sec (saves 22 minutes).

Reviewed By: Bastien Montagne
Differential Revision: https://developer.blender.org/D15215

2 files changed, 27 insertions, 16 deletions

diff --git a/source/blender/io/alembic/intern/alembic_capi.cc b/source/blender/io/alembic/intern/alembic_capi.cc
index 0d4e1d04db0..1fb535a57f2 100644
--- a/source/blender/io/alembic/intern/alembic_capi.cc
+++ b/source/blender/io/alembic/intern/alembic_capi.cc
@@ -573,12 +573,10 @@ static void import_endjob(void *user_data)
 
   ImportJobData *data = static_cast<ImportJobData *>(user_data);
 
-  std::vector<AbcObjectReader *>::iterator iter;
-
   /* Delete objects on cancellation. */
   if (data->was_cancelled) {
-    for (iter = data->readers.begin(); iter != data->readers.end(); ++iter) {
-      Object *ob = (*iter)->object();
+    for (AbcObjectReader *reader : data->readers) {
+      Object *ob = reader->object();
 
       /* It's possible that cancellation occurred between the creation of
        * the reader and the creation of the Blender object. */
@@ -590,7 +588,6 @@ static void import_endjob(void *user_data)
     }
   }
   else {
-    /* Add object to scene. */
     Base *base;
     LayerCollection *lc;
     ViewLayer *view_layer = data->view_layer;
@@ -599,11 +596,17 @@ static void import_endjob(void *user_data)
 
     lc = BKE_layer_collection_get_active(view_layer);
 
-    for (iter = data->readers.begin(); iter != data->readers.end(); ++iter) {
-      Object *ob = (*iter)->object();
-
+    /* Add all objects to the collection (don't do sync for each object). */
+    BKE_layer_collection_resync_forbid();
+    for (AbcObjectReader *reader : data->readers) {
+      Object *ob = reader->object();
       BKE_collection_object_add(data->bmain, lc->collection, ob);
-
+    }
+    /* Sync the collection, and do view layer operations. */
+    BKE_layer_collection_resync_allow();
+    BKE_main_collection_sync(data->bmain);
+    for (AbcObjectReader *reader : data->readers) {
+      Object *ob = reader->object();
       base = BKE_view_layer_base_find(view_layer, ob);
       /* TODO: is setting active needed? */
       BKE_view_layer_base_select_and_set_active(view_layer, base);
@@ -625,8 +628,7 @@ static void import_endjob(void *user_data)
     }
   }
 
-  for (iter = data->readers.begin(); iter != data->readers.end(); ++iter) {
-    AbcObjectReader *reader = *iter;
+  for (AbcObjectReader *reader : data->readers) {
     reader->decref();
 
     if (reader->refcount() == 0) {
diff --git a/source/blender/io/usd/intern/usd_capi_import.cc b/source/blender/io/usd/intern/usd_capi_import.cc
index 29b256125f0..4118205d87f 100644
--- a/source/blender/io/usd/intern/usd_capi_import.cc
+++ b/source/blender/io/usd/intern/usd_capi_import.cc
@@ -277,7 +277,6 @@ static void import_endjob(void *customdata)
     }
   }
   else if (data->archive) {
-    /* Add object to scene. */
     Base *base;
     LayerCollection *lc;
     ViewLayer *view_layer = data->view_layer;
@@ -286,20 +285,30 @@ static void import_endjob(void *customdata)
 
     lc = BKE_layer_collection_get_active(view_layer);
 
+    /* Add all objects to the collection (don't do sync for each object). */
+    BKE_layer_collection_resync_forbid();
     for (USDPrimReader *reader : data->archive->readers()) {
-
       if (!reader) {
         continue;
       }
-
       Object *ob = reader->object();
-
       if (!ob) {
         continue;
       }
-
       BKE_collection_object_add(data->bmain, lc->collection, ob);
+    }
 
+    /* Sync the collection, and do view layer operations. */
+    BKE_layer_collection_resync_allow();
+    BKE_main_collection_sync(data->bmain);
+    for (USDPrimReader *reader : data->archive->readers()) {
+      if (!reader) {
+        continue;
+      }
+      Object *ob = reader->object();
+      if (!ob) {
+        continue;
+      }
       base = BKE_view_layer_base_find(view_layer, ob);
       /* TODO: is setting active needed? */
       BKE_view_layer_base_select_and_set_active(view_layer, base);