Enhanced stats/reports for blendfile reading.

Add direct user feedback (as a warning report) to user when recursive
resync of overrides was needed.

And some timing (as CLOG logs) about main readfile process steps.

This is essentially adding a new BlendFileReadReport structure that wraps
BKE_reports list, and adds some extra info (some timing, some info about
overrides and (recursive) resync, etc.).

7 files changed, 71 insertions, 37 deletions

diff --git a/source/blender/blenloader/BLO_read_write.h b/source/blender/blenloader/BLO_read_write.h
index ea0532d884b..86c7c367816 100644
--- a/source/blender/blenloader/BLO_read_write.h
+++ b/source/blender/blenloader/BLO_read_write.h
@@ -54,6 +54,7 @@ typedef struct BlendExpander BlendExpander;
 typedef struct BlendLibReader BlendLibReader;
 typedef struct BlendWriter BlendWriter;
 
+struct BlendFileReadReport;
 struct Main;
 struct ReportList;
 
@@ -216,7 +217,7 @@ bool BLO_read_requires_endian_switch(BlendDataReader *reader);
 bool BLO_read_data_is_undo(BlendDataReader *reader);
 void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr);
 void BLO_read_glob_list(BlendDataReader *reader, struct ListBase *list);
-struct ReportList *BLO_read_data_reports(BlendDataReader *reader);
+struct BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader);
 
 /* Blend Read Lib API
  * ===================
@@ -233,7 +234,7 @@ ID *BLO_read_get_new_id_address(BlendLibReader *reader, struct Library *lib, str
 /* Misc. */
 bool BLO_read_lib_is_undo(BlendLibReader *reader);
 struct Main *BLO_read_lib_get_main(BlendLibReader *reader);
-struct ReportList *BLO_read_lib_reports(BlendLibReader *reader);
+struct BlendFileReadReport *BLO_read_lib_reports(BlendLibReader *reader);
 
 /* Blend Expand API
  * ===================
@@ -250,8 +251,10 @@ void BLO_expand_id(BlendExpander *expander, struct ID *id);
  * ===================
  */
 
-void BLO_reportf_wrap(struct ReportList *reports, ReportType type, const char *format, ...)
-    ATTR_PRINTF_FORMAT(3, 4);
+void BLO_reportf_wrap(struct BlendFileReadReport *reports,
+                      ReportType type,
+                      const char *format,
+                      ...) ATTR_PRINTF_FORMAT(3, 4);
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/blenloader/BLO_readfile.h b/source/blender/blenloader/BLO_readfile.h
index 89db216aada..62c3cb67091 100644
--- a/source/blender/blenloader/BLO_readfile.h
+++ b/source/blender/blenloader/BLO_readfile.h
@@ -89,6 +89,35 @@ struct BlendFileReadParams {
   int undo_direction; /* eUndoStepDir */
 };
 
+typedef struct BlendFileReadReport {
+  /* General reports handling. */
+  struct ReportList *reports;
+
+  /* Timing informations .*/
+  struct {
+    double whole;
+    double libraries;
+    double lib_overrides;
+    double lib_overrides_resync;
+    double lib_overrides_recursive_resync;
+  } duration;
+
+  /* Count informations. */
+  struct {
+    /* Some numbers of IDs that ended up in a specific state, or required some specific process
+     * during this file read. */
+    int missing_libraries;
+    int missing_linked_id;
+    /* Number of root override IDs that were resynced. */
+    int resynced_lib_overrides;
+  } count;
+
+  /* Number of libraries which had overrides that needed to be resynced, and a single linked list
+   * of those. */
+  int resynced_lib_overrides_libraries_count;
+  struct LinkNode *resynced_lib_overrides_libraries;
+} BlendFileReadReport;
+
 /* skip reading some data-block types (may want to skip screen data too). */
 typedef enum eBLOReadSkip {
   BLO_READ_SKIP_NONE = 0,
@@ -101,7 +130,7 @@ typedef enum eBLOReadSkip {
 
 BlendFileData *BLO_read_from_file(const char *filepath,
                                   eBLOReadSkip skip_flags,
-                                  struct ReportList *reports);
+                                  struct BlendFileReadReport *reports);
 BlendFileData *BLO_read_from_memory(const void *mem,
                                     int memsize,
                                     eBLOReadSkip skip_flags,
diff --git a/source/blender/blenloader/intern/readblenentry.c b/source/blender/blenloader/intern/readblenentry.c
index 19033ba9bf1..7131d8b887d 100644
--- a/source/blender/blenloader/intern/readblenentry.c
+++ b/source/blender/blenloader/intern/readblenentry.c
@@ -366,12 +366,12 @@ void BLO_blendhandle_close(BlendHandle *bh)
  */
 BlendFileData *BLO_read_from_file(const char *filepath,
                                   eBLOReadSkip skip_flags,
-                                  ReportList *reports)
+                                  BlendFileReadReport *reports)
 {
   BlendFileData *bfd = NULL;
   FileData *fd;
 
-  fd = blo_filedata_from_file(filepath, reports);
+  fd = blo_filedata_from_file(filepath, reports->reports);
   if (fd) {
     fd->reports = reports;
     fd->skip_flags = skip_flags;
@@ -401,7 +401,7 @@ BlendFileData *BLO_read_from_memory(const void *mem,
 
   fd = blo_filedata_from_memory(mem, memsize, reports);
   if (fd) {
-    fd->reports = reports;
+    fd->reports = &(struct BlendFileReadReport){.reports = reports};
     fd->skip_flags = skip_flags;
     bfd = blo_read_file_internal(fd, "");
     blo_filedata_free(fd);
@@ -430,7 +430,7 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain,
 
   fd = blo_filedata_from_memfile(memfile, params, reports);
   if (fd) {
-    fd->reports = reports;
+    fd->reports = &(struct BlendFileReadReport){.reports = reports};
     fd->skip_flags = params->skip_flags;
     BLI_strncpy(fd->relabase, filename, sizeof(fd->relabase));
 
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index 47ed4e5c06f..af214b81d52 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -72,6 +72,8 @@
 #include "BLI_mmap.h"
 #include "BLI_threads.h"
 
+#include "PIL_time.h"
+
 #include "BLT_translation.h"
 
 #include "BKE_anim_data.h"
@@ -227,7 +229,7 @@ typedef struct BHeadN {
  * bit kludge but better than doubling up on prints,
  * we could alternatively have a versions of a report function which forces printing - campbell
  */
-void BLO_reportf_wrap(ReportList *reports, ReportType type, const char *format, ...)
+void BLO_reportf_wrap(BlendFileReadReport *reports, ReportType type, const char *format, ...)
 {
   char fixed_buf[1024]; /* should be long enough */
 
@@ -239,7 +241,7 @@ void BLO_reportf_wrap(ReportList *reports, ReportType type, const char *format,
 
   fixed_buf[sizeof(fixed_buf) - 1] = '\0';
 
-  BKE_report(reports, type, fixed_buf);
+  BKE_report(reports->reports, type, fixed_buf);
 
   if (G.background == 0) {
     printf("%s: %s\n", BKE_report_type_str(type), fixed_buf);
@@ -4206,6 +4208,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
   }
 
   if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) {
+    fd->reports->duration.libraries = PIL_check_seconds_timer();
     read_libraries(fd, &mainlist);
 
     blo_join_main(&mainlist);
@@ -4213,6 +4216,8 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
     lib_link_all(fd, bfd->main);
     after_liblink_merged_bmain_process(bfd->main);
 
+    fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries;
+
     /* Skip in undo case. */
     if (fd->memfile == NULL) {
       /* Note that we can't recompute user-counts at this point in undo case, we play too much with
@@ -4228,7 +4233,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
       blo_split_main(&mainlist, bfd->main);
       LISTBASE_FOREACH (Main *, mainvar, &mainlist) {
         BLI_assert(mainvar->versionfile != 0);
-        do_versions_after_linking(mainvar, fd->reports);
+        do_versions_after_linking(mainvar, fd->reports->reports);
       }
       blo_join_main(&mainlist);
 
@@ -4249,8 +4254,13 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath)
      * we can re-generate overrides from their references. */
     if (fd->memfile == NULL) {
       /* Do not apply in undo case! */
-      BKE_lib_override_library_main_validate(bfd->main, fd->reports);
+      fd->reports->duration.lib_overrides = PIL_check_seconds_timer();
+
+      BKE_lib_override_library_main_validate(bfd->main, fd->reports->reports);
       BKE_lib_override_library_main_update(bfd->main);
+
+      fd->reports->duration.lib_overrides = PIL_check_seconds_timer() -
+                                            fd->reports->duration.lib_overrides;
     }
 
     BKE_collections_after_lib_link(bfd->main);
@@ -5206,7 +5216,7 @@ static void library_link_end(Main *mainl,
      * or they will go again through do_versions - bad, very bad! */
     split_main_newid(mainvar, main_newid);
 
-    do_versions_after_linking(main_newid, (*fd)->reports);
+    do_versions_after_linking(main_newid, (*fd)->reports->reports);
 
     add_main_to_main(mainvar, main_newid);
   }
@@ -5340,7 +5350,7 @@ static void read_library_linked_id(
                      id->name + 2,
                      mainvar->curlib->filepath_abs,
                      library_parent_filepath(mainvar->curlib));
-    basefd->library_id_missing_count++;
+    basefd->reports->count.missing_linked_id++;
 
     /* Generate a placeholder for this ID (simplified version of read_libblock actually...). */
     if (r_id) {
@@ -5444,7 +5454,7 @@ static FileData *read_library_file_data(FileData *basefd,
                      TIP_("Read packed library:  '%s', parent '%s'"),
                      mainptr->curlib->filepath,
                      library_parent_filepath(mainptr->curlib));
-    fd = blo_filedata_from_memory(pf->data, pf->size, basefd->reports);
+    fd = blo_filedata_from_memory(pf->data, pf->size, basefd->reports->reports);
 
     /* Needed for library_append and read_libraries. */
     BLI_strncpy(fd->relabase, mainptr->curlib->filepath_abs, sizeof(fd->relabase));
@@ -5457,7 +5467,7 @@ static FileData *read_library_file_data(FileData *basefd,
                      mainptr->curlib->filepath_abs,
                      mainptr->curlib->filepath,
                      library_parent_filepath(mainptr->curlib));
-    fd = blo_filedata_from_file(mainptr->curlib->filepath_abs, basefd->reports);
+    fd = blo_filedata_from_file(mainptr->curlib->filepath_abs, basefd->reports->reports);
   }
 
   if (fd) {
@@ -5495,7 +5505,7 @@ static FileData *read_library_file_data(FileData *basefd,
   if (fd == NULL) {
     BLO_reportf_wrap(
         basefd->reports, RPT_INFO, TIP_("Cannot find lib '%s'"), mainptr->curlib->filepath_abs);
-    basefd->library_file_missing_count++;
+    basefd->reports->count.missing_libraries++;
   }
 
   return fd;
@@ -5589,15 +5599,6 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
     mainptr->curlib->filedata = NULL;
   }
   BKE_main_free(main_newid);
-
-  if (basefd->library_file_missing_count != 0 || basefd->library_id_missing_count != 0) {
-    BKE_reportf(basefd->reports,
-                RPT_WARNING,
-                "LIB: %d libraries and %d linked data-blocks are missing, please check the "
-                "Info and Outliner editors for details",
-                basefd->library_file_missing_count,
-                basefd->library_id_missing_count);
-  }
 }
 
 void *BLO_read_get_new_data_address(BlendDataReader *reader, const void *old_address)
@@ -5785,7 +5786,7 @@ void BLO_read_glob_list(BlendDataReader *reader, ListBase *list)
   link_glob_list(reader->fd, list);
 }
 
-ReportList *BLO_read_data_reports(BlendDataReader *reader)
+BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader)
 {
   return reader->fd->reports;
 }
@@ -5800,7 +5801,7 @@ Main *BLO_read_lib_get_main(BlendLibReader *reader)
   return reader->main;
 }
 
-ReportList *BLO_read_lib_reports(BlendLibReader *reader)
+BlendFileReadReport *BLO_read_lib_reports(BlendLibReader *reader)
 {
   return reader->fd->reports;
 }
diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h
index d1d4e0b3256..0fc71760b44 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -145,11 +145,7 @@ typedef struct FileData {
   ListBase *old_mainlist;
   struct IDNameLib_Map *old_idmap;
 
-  struct ReportList *reports;
-  /* Counters for amount of missing libraries, and missing IDs in libraries.
-   * Used to generate a synthetic report in the UI. */
-  int library_file_missing_count;
-  int library_id_missing_count;
+  struct BlendFileReadReport *reports;
 } FileData;
 
 #define SIZEOFBLENDERHEADER 12
diff --git a/source/blender/blenloader/intern/versioning_280.c b/source/blender/blenloader/intern/versioning_280.c
index bf8f7eeca5c..0059074c8ad 100644
--- a/source/blender/blenloader/intern/versioning_280.c
+++ b/source/blender/blenloader/intern/versioning_280.c
@@ -1912,7 +1912,9 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain)
     FOREACH_NODETREE_END;
 
     if (error & NTREE_DOVERSION_NEED_OUTPUT) {
-      BKE_report(fd->reports, RPT_ERROR, "Eevee material conversion problem. Error in console");
+      BKE_report(fd->reports != NULL ? fd->reports->reports : NULL,
+                 RPT_ERROR,
+                 "Eevee material conversion problem. Error in console");
       printf(
           "You need to connect Principled and Eevee Specular shader nodes to new material "
           "output "
@@ -1920,7 +1922,9 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain)
     }
 
     if (error & NTREE_DOVERSION_TRANSPARENCY_EMISSION) {
-      BKE_report(fd->reports, RPT_ERROR, "Eevee material conversion problem. Error in console");
+      BKE_report(fd->reports != NULL ? fd->reports->reports : NULL,
+                 RPT_ERROR,
+                 "Eevee material conversion problem. Error in console");
       printf(
           "You need to combine transparency and emission shaders to the converted Principled "
           "shader nodes.\n");
diff --git a/source/blender/blenloader/tests/blendfile_loading_base_test.cc b/source/blender/blenloader/tests/blendfile_loading_base_test.cc
index 280a4b42b36..e93348b2158 100644
--- a/source/blender/blenloader/tests/blendfile_loading_base_test.cc
+++ b/source/blender/blenloader/tests/blendfile_loading_base_test.cc
@@ -123,7 +123,8 @@ bool BlendfileLoadingBaseTest::blendfile_load(const char *filepath)
   char abspath[FILENAME_MAX];
   BLI_path_join(abspath, sizeof(abspath), test_assets_dir.c_str(), filepath, NULL);
 
-  bfile = BLO_read_from_file(abspath, BLO_READ_SKIP_NONE, nullptr /* reports */);
+  BlendFileReadReport bf_reports = {NULL};
+  bfile = BLO_read_from_file(abspath, BLO_READ_SKIP_NONE, &bf_reports);
   if (bfile == nullptr) {
     ADD_FAILURE() << "Unable to load file '" << filepath << "' from test assets dir '"
                   << test_assets_dir << "'";