Speed up the new OBJ exporter via bigger write buffer and parallelization.

This is a patch from Aras Pranckevicius, D13927. See that patch for full
details. On Windows, the many small fprintfs were taking up a large amount
of time and significant speedup comes from using snprintf into chained buffers,
and writing them all out later.
On both Windows and Linux, parallelizing the processing by Object can also lead
to a significant increase in speed.
The 3.0 splash screen scene exports 8 times faster than the current C++ exporter
on a Windows machine with 32 threads, and 5.8 times faster on a Linux machine
with 48 threads.

There is admittedly more memory usage for this, but it is still using 25 times
less memory than the old python exporter on the 3.0 splash screen scene, so
this seems an acceptable tradeoff. If use cases come up for exporting obj files
that exceed the memory size of users, a flag could be added to not parallelize
and write the buffers out every so often.

1 files changed, 83 insertions, 27 deletions

diff --git a/source/blender/io/wavefront_obj/exporter/obj_exporter.cc b/source/blender/io/wavefront_obj/exporter/obj_exporter.cc
index 0c753ccdcac..187f50277f1 100644
--- a/source/blender/io/wavefront_obj/exporter/obj_exporter.cc
+++ b/source/blender/io/wavefront_obj/exporter/obj_exporter.cc
@@ -25,6 +25,7 @@
 #include "BKE_scene.h"
 
 #include "BLI_path_util.h"
+#include "BLI_task.hh"
 #include "BLI_vector.hh"
 
 #include "DEG_depsgraph_query.h"
@@ -155,44 +156,97 @@ static void write_mesh_objects(Vector<std::unique_ptr<OBJMesh>> exportable_as_me
                                MTLWriter *mtl_writer,
                                const OBJExportParams &export_params)
 {
+  /* Parallelization is over meshes/objects, which means
+   * we have to have the output text buffer for each object,
+   * and write them all into the file at the end. */
+  size_t count = exportable_as_mesh.size();
+  std::vector<FormatHandler<eFileType::OBJ>> buffers(count);
+
+  /* Serial: gather material indices, ensure normals & edges. */
+  Vector<Vector<int>> mtlindices;
   if (mtl_writer) {
     obj_writer.write_mtllib_name(mtl_writer->mtl_file_path());
+    mtlindices.reserve(count);
   }
-
-  /* Smooth groups and UV vertex indices may make huge memory allocations, so they should be freed
-   * right after they're written, instead of waiting for #blender::Vector to clean them up after
-   * all the objects are exported. */
   for (auto &obj_mesh : exportable_as_mesh) {
-    obj_writer.write_object_name(*obj_mesh);
-    obj_writer.write_vertex_coords(*obj_mesh);
-    Vector<int> obj_mtlindices;
+    OBJMesh &obj = *obj_mesh;
+    if (mtl_writer) {
+      mtlindices.append(mtl_writer->add_materials(obj));
+    }
+    if (export_params.export_normals) {
+      obj.ensure_mesh_normals();
+    }
+    obj.ensure_mesh_edges();
+  }
 
-    if (obj_mesh->tot_polygons() > 0) {
-      if (export_params.export_smooth_groups) {
-        obj_mesh->calc_smooth_groups(export_params.smooth_groups_bitflags);
-      }
+  /* Parallel over meshes: store normal coords & indices, uv coords and indices. */
+  blender::threading::parallel_for(IndexRange(count), 1, [&](IndexRange range) {
+    for (const int i : range) {
+      OBJMesh &obj = *exportable_as_mesh[i];
       if (export_params.export_normals) {
-        obj_writer.write_poly_normals(*obj_mesh);
+        obj.store_normal_coords_and_indices();
       }
       if (export_params.export_uv) {
-        obj_writer.write_uv_coords(*obj_mesh);
-      }
-      if (mtl_writer) {
-        obj_mtlindices = mtl_writer->add_materials(*obj_mesh);
+        obj.store_uv_coords_and_indices();
       }
-      /* This function takes a 0-indexed slot index for the obj_mesh object and
-       * returns the material name that we are using in the .obj file for it. */
-      std::function<const char *(int)> matname_fn = [&](int s) -> const char * {
-        if (!mtl_writer || s < 0 || s >= obj_mtlindices.size()) {
-          return nullptr;
+    }
+  });
+
+  /* Serial: calculate index offsets; these are sequentially added
+   * over all meshes, and requite normal/uv indices to be calculated. */
+  Vector<IndexOffsets> index_offsets;
+  index_offsets.reserve(count);
+  IndexOffsets offsets{0, 0, 0};
+  for (auto &obj_mesh : exportable_as_mesh) {
+    OBJMesh &obj = *obj_mesh;
+    index_offsets.append(offsets);
+    offsets.vertex_offset += obj.tot_vertices();
+    offsets.uv_vertex_offset += obj.tot_uv_vertices();
+    offsets.normal_offset += obj.tot_normal_indices();
+  }
+
+  /* Parallel over meshes: main result writing. */
+  blender::threading::parallel_for(IndexRange(count), 1, [&](IndexRange range) {
+    for (const int i : range) {
+      OBJMesh &obj = *exportable_as_mesh[i];
+      auto &fh = buffers[i];
+
+      obj_writer.write_object_name(fh, obj);
+      obj_writer.write_vertex_coords(fh, obj);
+
+      if (obj.tot_polygons() > 0) {
+        if (export_params.export_smooth_groups) {
+          obj.calc_smooth_groups(export_params.smooth_groups_bitflags);
+        }
+        if (export_params.export_normals) {
+          obj_writer.write_poly_normals(fh, obj);
         }
-        return mtl_writer->mtlmaterial_name(obj_mtlindices[s]);
-      };
-      obj_writer.write_poly_elements(*obj_mesh, matname_fn);
+        if (export_params.export_uv) {
+          obj_writer.write_uv_coords(fh, obj);
+        }
+        /* This function takes a 0-indexed slot index for the obj_mesh object and
+         * returns the material name that we are using in the .obj file for it. */
+        const auto *obj_mtlindices = mtlindices.is_empty() ? nullptr : &mtlindices[i];
+        std::function<const char *(int)> matname_fn = [&](int s) -> const char * {
+          if (!obj_mtlindices || s < 0 || s >= obj_mtlindices->size()) {
+            return nullptr;
+          }
+          return mtl_writer->mtlmaterial_name((*obj_mtlindices)[s]);
+        };
+        obj_writer.write_poly_elements(fh, index_offsets[i], obj, matname_fn);
+      }
+      obj_writer.write_edges_indices(fh, index_offsets[i], obj);
+
+      /* Nothing will need this object's data after this point, release
+       * various arrays here. */
+      obj.clear();
     }
-    obj_writer.write_edges_indices(*obj_mesh);
+  });
 
-    obj_writer.update_index_offsets(*obj_mesh);
+  /* Write all the object text buffers into the output file. */
+  FILE *f = obj_writer.get_outfile();
+  for (auto &b : buffers) {
+    b.write_to_file(f);
   }
 }
 
@@ -202,11 +256,13 @@ static void write_mesh_objects(Vector<std::unique_ptr<OBJMesh>> exportable_as_me
 static void write_nurbs_curve_objects(const Vector<std::unique_ptr<OBJCurve>> &exportable_as_nurbs,
                                       const OBJWriter &obj_writer)
 {
+  FormatHandler<eFileType::OBJ> fh;
   /* #OBJCurve doesn't have any dynamically allocated memory, so it's fine
    * to wait for #blender::Vector to clean the objects up. */
   for (const std::unique_ptr<OBJCurve> &obj_curve : exportable_as_nurbs) {
-    obj_writer.write_nurbs_curve(*obj_curve);
+    obj_writer.write_nurbs_curve(fh, *obj_curve);
   }
+  fh.write_to_file(obj_writer.get_outfile());
 }
 
 void export_frame(Depsgraph *depsgraph, const OBJExportParams &export_params, const char *filepath)