DNA: optimize struct reconstruction by doing some preprocessing

When loading large files that are more than a couple weeks old
(such that DNA has changed in that time), a significant amount of
time is spent in `DNA_struct_reconstruct`. This function takes a struct
in the old layout and creates a struct in the new layout from it.

This was slow because it was computing the diff between the struct
layouts every time a struct is updated. Now the steps for the struct
reconstruction is computed only once per struct. This information is
then used to actually reconstruct all structs that changed.

I measured about 10-20% speedup when loading Spring files.
E.g. `10.6s -> 8.7s` for `06_055_A.anim.blend` in BKE_blendfile_read`.

This percentage varies a lot based on the number of blocks that have
to be reconstructed and how much DNA has changed since they have
been written. In none of my tests was the new code slower than the
old code.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D8959

2 files changed, 9 insertions, 4 deletions

diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 19fab73b259..bd944ac32ac 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -779,7 +779,7 @@ static void bh4_from_bh8(BHead *bhead, BHead8 *bhead8, int do_endian_swap)
      * 0x0000000000000000000012345678 would become 0x12345678000000000000000000000000
      */
     if (do_endian_swap) {
-      BLI_endian_switch_int64(&bhead8->old);
+      BLI_endian_switch_uint64(&bhead8->old);
     }
 
     /* this patch is to avoid a long long being read from not-eight aligned positions
@@ -1114,6 +1114,8 @@ static bool read_file_dna(FileData *fd, const char **r_error_message)
       if (fd->filesdna) {
         blo_do_versions_dna(fd->filesdna, fd->fileversion, subversion);
         fd->compflags = DNA_struct_get_compareflags(fd->filesdna, fd->memsdna);
+        fd->reconstruct_info = DNA_reconstruct_info_create(
+            fd->filesdna, fd->memsdna, fd->compflags);
         /* used to retrieve ID names from (bhead+1) */
         fd->id_name_offs = DNA_elem_offset(fd->filesdna, "ID", "char", "name[]");
 
@@ -1610,6 +1612,9 @@ void blo_filedata_free(FileData *fd)
     if (fd->compflags) {
       MEM_freeN((void *)fd->compflags);
     }
+    if (fd->reconstruct_info) {
+      DNA_reconstruct_info_free(fd->reconstruct_info);
+    }
 
     if (fd->datamap) {
       oldnewmap_free(fd->datamap);
@@ -2177,8 +2182,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname)
           }
         }
 #endif
-        temp = DNA_struct_reconstruct(
-            fd->memsdna, fd->filesdna, fd->compflags, bh->SDNAnr, bh->nr, (bh + 1));
+        temp = DNA_struct_reconstruct(fd->reconstruct_info, bh->SDNAnr, bh->nr, (bh + 1));
       }
       else {
         /* SDNA_CMP_EQUAL */
@@ -9135,7 +9139,7 @@ static void convert_pointer_array_32_to_64(BlendDataReader *UNUSED(reader),
                                            const uint32_t *src,
                                            uint64_t *dst)
 {
-  /* Match pointer conversion rules from bh8_from_bh4 and cast_pointer. */
+  /* Match pointer conversion rules from bh8_from_bh4 and cast_pointer_32_to_64. */
   for (int i = 0; i < array_size; i++) {
     dst[i] = src[i];
   }
diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h
index d43f7ded50e..2ddf96a2d47 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -106,6 +106,7 @@ typedef struct FileData {
   const struct SDNA *memsdna;
   /** Array of #eSDNA_StructCompare. */
   const char *compflags;
+  struct DNA_ReconstructInfo *reconstruct_info;
 
   int fileversion;
   /** Used to retrieve ID names from (bhead+1). */