FileBrowser: Reduce Overhead Browsing Libraries.

When Browsing libraries the asset files were opened multiple times.
once to determine the needed groups to query and once for each
group to query the items in the group. For file browsing this makes sense
but for asset browsing this can be reduced.

This patch will load the asset files recursively and only opens them once.
Another change is that only the assets are requested and not filtered out
later in the process.

This patch is needed to simplify the library indexing. Where
we need access to the full library content.

## The numbers ##

Benchmarked by adding scenes of the spring open movie to the default
asset library. Refreshing the asset library would recursively load all the files
there.

| **8bc27c508a** | Processed 317 'directories/libraries' | 7.573986s |
| **Patch** | Processed 42 'directories/libraries' | 0.821013s |

{F10442811}

Reviewed By: mont29, Severin

Maniphest Tasks: T91406

Differential Revision: https://developer.blender.org/D12499

3 files changed, 187 insertions, 89 deletions

diff --git a/source/blender/blenloader/BLO_readfile.h b/source/blender/blenloader/BLO_readfile.h
index c8615545df9..4b4311d32bf 100644
--- a/source/blender/blenloader/BLO_readfile.h
+++ b/source/blender/blenloader/BLO_readfile.h
@@ -168,9 +168,8 @@ struct LinkNode *BLO_blendhandle_get_datablock_names(BlendHandle *bh,
 
                                                      const bool use_assets_only,
                                                      int *r_tot_names);
-struct LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh,
-                                                    int ofblocktype,
-                                                    int *r_tot_info_items);
+struct LinkNode * /*BLODataBlockInfo */ BLO_blendhandle_get_datablock_info(
+    BlendHandle *bh, int ofblocktype, const bool use_assets_only, int *r_tot_info_items);
 struct LinkNode *BLO_blendhandle_get_previews(BlendHandle *bh, int ofblocktype, int *r_tot_prev);
 struct PreviewImage *BLO_blendhandle_get_preview_for_id(BlendHandle *bh,
                                                         int ofblocktype,
diff --git a/source/blender/blenloader/intern/readblenentry.c b/source/blender/blenloader/intern/readblenentry.c
index f88b470809c..3306eb9e454 100644
--- a/source/blender/blenloader/intern/readblenentry.c
+++ b/source/blender/blenloader/intern/readblenentry.c
@@ -170,17 +170,19 @@ LinkNode *BLO_blendhandle_get_datablock_names(BlendHandle *bh,
 }
 
 /**
- * Gets the names and asset-data (if ID is an asset) of all the data-blocks in a file of a certain
- * type (e.g. all the scene names in a file).
+ * Gets the names and asset-data (if ID is an asset) of data-blocks in a file of a certain type.
+ * The data-blocks can be limited to assets.
  *
  * \param bh: The blendhandle to access.
  * \param ofblocktype: The type of names to get.
+ * \param use_assets_only: Limit the result to assets only.
  * \param tot_info_items: The length of the returned list.
  * \return A BLI_linklist of BLODataBlockInfo *. The links and #BLODataBlockInfo.asset_data should
  *         be freed with MEM_freeN.
  */
 LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh,
                                              int ofblocktype,
+                                             const bool use_assets_only,
                                              int *r_tot_info_items)
 {
   FileData *fd = (FileData *)bh;
@@ -189,27 +191,34 @@ LinkNode *BLO_blendhandle_get_datablock_info(BlendHandle *bh,
   int tot = 0;
 
   for (bhead = blo_bhead_first(fd); bhead; bhead = blo_bhead_next(fd, bhead)) {
+    if (bhead->code == ENDB) {
+      break;
+    }
     if (bhead->code == ofblocktype) {
-      struct BLODataBlockInfo *info = MEM_mallocN(sizeof(*info), __func__);
       const char *name = blo_bhead_id_name(fd, bhead) + 2;
+      AssetMetaData *asset_meta_data = blo_bhead_id_asset_data_address(fd, bhead);
 
-      STRNCPY(info->name, name);
+      const bool is_asset = asset_meta_data != NULL;
+      const bool skip_datablock = use_assets_only && !is_asset;
+      if (skip_datablock) {
+        continue;
+      }
+      struct BLODataBlockInfo *info = MEM_mallocN(sizeof(*info), __func__);
 
       /* Lastly, read asset data from the following blocks. */
-      info->asset_data = blo_bhead_id_asset_data_address(fd, bhead);
-      if (info->asset_data) {
-        bhead = blo_read_asset_data_block(fd, bhead, &info->asset_data);
-        /* blo_read_asset_data_block() reads all DATA heads and already advances bhead to the next
-         * non-DATA one. Go back, so the loop doesn't skip the non-DATA head. */
+      if (asset_meta_data) {
+        bhead = blo_read_asset_data_block(fd, bhead, &asset_meta_data);
+        /* blo_read_asset_data_block() reads all DATA heads and already advances bhead to the
+         * next non-DATA one. Go back, so the loop doesn't skip the non-DATA head. */
         bhead = blo_bhead_prev(fd, bhead);
       }
 
+      STRNCPY(info->name, name);
+      info->asset_data = asset_meta_data;
+
       BLI_linklist_prepend(&infos, info);
       tot++;
     }
-    else if (bhead->code == ENDB) {
-      break;
-    }
   }
 
   *r_tot_info_items = tot;
diff --git a/source/blender/editors/space_file/filelist.c b/source/blender/editors/space_file/filelist.c
index 4f881184990..5c0976e18f2 100644
--- a/source/blender/editors/space_file/filelist.c
+++ b/source/blender/editors/space_file/filelist.c
@@ -2888,76 +2888,129 @@ static int filelist_readjob_list_dir(const char *root,
   return nbr_entries;
 }
 
-static int filelist_readjob_list_lib(const char *root, ListBase *entries, const bool skip_currpar)
+typedef enum ListLibOptions {
+  /* Will read both the groups + actual ids from the library. Reduces the amount of times that
+   * a library needs to be opened. */
+  LIST_LIB_RECURSIVE = (1 << 0),
+
+  /* Will only list assets. */
+  LIST_LIB_ASSETS_ONLY = (1 << 1),
+
+  /* Add given root as result. */
+  LIST_LIB_ADD_PARENT = (1 << 2),
+} ListLibOptions;
+
+static FileListInternEntry *filelist_readjob_list_lib_group_create(const int idcode,
+                                                                   const char *group_name)
+{
+  FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
+  entry->relpath = BLI_strdup(group_name);
+  entry->typeflag |= FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR;
+  entry->blentype = idcode;
+  return entry;
+}
+
+static void filelist_readjob_list_lib_add_datablocks(ListBase *entries,
+                                                     LinkNode *datablock_infos,
+                                                     const bool prefix_relpath_with_group_name,
+                                                     const int idcode,
+                                                     const char *group_name)
+{
+  for (LinkNode *ln = datablock_infos; ln; ln = ln->next) {
+    struct BLODataBlockInfo *info = ln->link;
+    FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
+    if (prefix_relpath_with_group_name) {
+      entry->relpath = BLI_sprintfN("%s/%s", group_name, info->name);
+    }
+    else {
+      entry->relpath = BLI_strdup(info->name);
+    }
+    entry->typeflag |= FILE_TYPE_BLENDERLIB;
+    if (info && info->asset_data) {
+      entry->typeflag |= FILE_TYPE_ASSET;
+      /* Moves ownership! */
+      entry->imported_asset_data = info->asset_data;
+    }
+    entry->blentype = idcode;
+    BLI_addtail(entries, entry);
+  }
+}
+
+static int filelist_readjob_list_lib(const char *root,
+                                     ListBase *entries,
+                                     const ListLibOptions options)
 {
-  FileListInternEntry *entry;
-  LinkNode *ln, *names = NULL, *datablock_infos = NULL;
-  int i, nitems, idcode = 0, nbr_entries = 0;
   char dir[FILE_MAX_LIBEXTRA], *group;
-  bool ok;
 
   struct BlendHandle *libfiledata = NULL;
 
-  /* name test */
-  ok = BLO_library_path_explode(root, dir, &group, NULL);
-  if (!ok) {
-    return nbr_entries;
+  /* Check if the given root is actually a library. All folders are passed to
+   * `filelist_readjob_list_lib` and based on the number of found entries `filelist_readjob_do`
+   * will do a dir listing only when this function does not return any entries. */
+  /* TODO: We should consider introducing its own function to detect if it is a lib and
+   * call it directly from `filelist_readjob_do` to increase readability. */
+  const bool is_lib = BLO_library_path_explode(root, dir, &group, NULL);
+  if (!is_lib) {
+    return 0;
   }
 
-  /* there we go */
+  /* Open the library file. */
   BlendFileReadReport bf_reports = {.reports = NULL};
   libfiledata = BLO_blendhandle_from_file(dir, &bf_reports);
   if (libfiledata == NULL) {
-    return nbr_entries;
-  }
-
-  /* memory for strings is passed into filelist[i].entry->relpath
-   * and freed in filelist_entry_free. */
-  if (group) {
-    idcode = groupname_to_code(group);
-    datablock_infos = BLO_blendhandle_get_datablock_info(libfiledata, idcode, &nitems);
-  }
-  else {
-    names = BLO_blendhandle_get_linkable_groups(libfiledata);
-    nitems = BLI_linklist_count(names);
+    return 0;
   }
 
-  BLO_blendhandle_close(libfiledata);
-
-  if (!skip_currpar) {
-    entry = MEM_callocN(sizeof(*entry), __func__);
+  /* Add current parent when requested. */
+  int parent_len = 0;
+  if (options & LIST_LIB_ADD_PARENT) {
+    FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__);
     entry->relpath = BLI_strdup(FILENAME_PARENT);
     entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR);
     BLI_addtail(entries, entry);
-    nbr_entries++;
+    parent_len = 1;
   }
 
-  for (i = 0, ln = (datablock_infos ? datablock_infos : names); i < nitems; i++, ln = ln->next) {
-    struct BLODataBlockInfo *info = datablock_infos ? ln->link : NULL;
-    const char *blockname = info ? info->name : ln->link;
-
-    entry = MEM_callocN(sizeof(*entry), __func__);
-    entry->relpath = BLI_strdup(blockname);
-    entry->typeflag |= FILE_TYPE_BLENDERLIB;
-    if (info && info->asset_data) {
-      entry->typeflag |= FILE_TYPE_ASSET;
-      /* Moves ownership! */
-      entry->imported_asset_data = info->asset_data;
-    }
-    if (!(group && idcode)) {
-      entry->typeflag |= FILE_TYPE_DIR;
-      entry->blentype = groupname_to_code(blockname);
-    }
-    else {
-      entry->blentype = idcode;
+  int group_len = 0;
+  int datablock_len = 0;
+  const bool group_came_from_path = group != NULL;
+  if (group_came_from_path) {
+    const int idcode = groupname_to_code(group);
+    LinkNode *datablock_infos = BLO_blendhandle_get_datablock_info(
+        libfiledata, idcode, options & LIST_LIB_ASSETS_ONLY, &datablock_len);
+    filelist_readjob_list_lib_add_datablocks(entries, datablock_infos, false, idcode, group);
+    BLI_linklist_freeN(datablock_infos);
+  }
+  else {
+    LinkNode *groups = BLO_blendhandle_get_linkable_groups(libfiledata);
+    group_len = BLI_linklist_count(groups);
+
+    for (LinkNode *ln = groups; ln; ln = ln->next) {
+      const char *group_name = ln->link;
+      const int idcode = groupname_to_code(group_name);
+      FileListInternEntry *group_entry = filelist_readjob_list_lib_group_create(idcode,
+                                                                                group_name);
+      BLI_addtail(entries, group_entry);
+
+      if (options & LIST_LIB_RECURSIVE) {
+        int group_datablock_len;
+        LinkNode *group_datablock_infos = BLO_blendhandle_get_datablock_info(
+            libfiledata, idcode, options & LIST_LIB_ASSETS_ONLY, &group_datablock_len);
+        filelist_readjob_list_lib_add_datablocks(
+            entries, group_datablock_infos, true, idcode, group_name);
+        BLI_linklist_freeN(group_datablock_infos);
+        datablock_len += group_datablock_len;
+      }
     }
-    BLI_addtail(entries, entry);
-    nbr_entries++;
+
+    BLI_linklist_freeN(groups);
   }
 
-  BLI_linklist_freeN(datablock_infos ? datablock_infos : names);
+  BLO_blendhandle_close(libfiledata);
 
-  return nbr_entries;
+  /* Return the number of items added to entries. */
+  int added_entries_len = group_len + datablock_len + parent_len;
+  return added_entries_len;
 }
 
 #if 0
@@ -3153,6 +3206,35 @@ typedef struct FileListReadJob {
   struct FileList *tmp_filelist;
 } FileListReadJob;
 
+static bool filelist_readjob_should_recurse_into_entry(const int max_recursion,
+                                                       const int current_recursion_level,
+                                                       FileListInternEntry *entry)
+{
+  if (max_recursion == 0) {
+    /* Recursive loading is disabled. */
+    return false;
+  }
+  if (current_recursion_level >= max_recursion) {
+    /* No more levels of recursion left. */
+    return false;
+  }
+  if (entry->typeflag & FILE_TYPE_BLENDERLIB) {
+    /* Libraries are already loaded recursively when recursive loaded is used. No need to add
+     * them another time. This loading is done with the `LIST_LIB_RECURSIVE` option. */
+    return false;
+  }
+  if (!(entry->typeflag & FILE_TYPE_DIR)) {
+    /* Cannot recurse into regular file entries. */
+    return false;
+  }
+  if (FILENAME_IS_CURRPAR(entry->relpath)) {
+    /* Don't schedule go to parent entry, (`..`) */
+    return false;
+  }
+
+  return true;
+}
+
 static void filelist_readjob_do(const bool do_lib,
                                 FileListReadJob *job_params,
                                 const short *stop,
@@ -3189,7 +3271,6 @@ static void filelist_readjob_do(const bool do_lib,
   while (!BLI_stack_is_empty(todo_dirs) && !(*stop)) {
     FileListInternEntry *entry;
     int nbr_entries = 0;
-    bool is_lib = do_lib;
 
     char *subdir;
     char rel_subdir[FILE_MAX_LIBEXTRA];
@@ -3212,45 +3293,54 @@ static void filelist_readjob_do(const bool do_lib,
     BLI_path_normalize_dir(root, rel_subdir);
     BLI_path_rel(rel_subdir, root);
 
+    bool is_lib = false;
     if (do_lib) {
-      nbr_entries = filelist_readjob_list_lib(subdir, &entries, skip_currpar);
+      ListLibOptions list_lib_options = 0;
+      if (!skip_currpar) {
+        list_lib_options |= LIST_LIB_ADD_PARENT;
+      }
+
+      /* Libraries are loaded recursively when max_recursion is set. It doesn't check if there is
+       * still a recursion level over. */
+      if (max_recursion > 0) {
+        list_lib_options |= LIST_LIB_RECURSIVE;
+      }
+      /* Only load assets when browsing an asset library. For normal file browsing we return all
+       * entries. `FLF_ASSETS_ONLY` filter can be enabled/disabled by the user.*/
+      if (filelist->asset_library_ref) {
+        list_lib_options |= LIST_LIB_ASSETS_ONLY;
+      }
+      nbr_entries = filelist_readjob_list_lib(subdir, &entries, list_lib_options);
+      if (nbr_entries > 0) {
+        is_lib = true;
+      }
     }
-    if (!nbr_entries) {
-      is_lib = false;
+
+    if (!is_lib) {
       nbr_entries = filelist_readjob_list_dir(
           subdir, &entries, filter_glob, do_lib, job_params->main_name, skip_currpar);
     }
 
     for (entry = entries.first; entry; entry = entry->next) {
-      BLI_join_dirfile(dir, sizeof(dir), rel_subdir, entry->relpath);
-
       entry->uid = filelist_uid_generate(filelist);
 
-      /* Only thing we change in direntry here, so we need to free it first. */
+      /* When loading entries recursive, the rel_path should be relative from the root dir.
+       * we combine the relative path to the subdir with the relative path of the entry. */
+      BLI_join_dirfile(dir, sizeof(dir), rel_subdir, entry->relpath);
       MEM_freeN(entry->relpath);
       entry->relpath = BLI_strdup(dir + 2); /* + 2 to remove '//'
                                              * added by BLI_path_rel to rel_subdir. */
       entry->name = fileentry_uiname(root, entry->relpath, entry->typeflag, dir);
       entry->free_name = true;
 
-      /* Here we decide whether current filedirentry is to be listed too, or not. */
-      if (max_recursion && (is_lib || (recursion_level <= max_recursion))) {
-        if (((entry->typeflag & FILE_TYPE_DIR) == 0) || FILENAME_IS_CURRPAR(entry->relpath)) {
-          /* Skip... */
-        }
-        else if (!is_lib && (recursion_level >= max_recursion) &&
-                 ((entry->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) == 0)) {
-          /* Do not recurse in real directories in this case, only in .blend libs. */
-        }
-        else {
-          /* We have a directory we want to list, add it to todo list! */
-          BLI_join_dirfile(dir, sizeof(dir), root, entry->relpath);
-          BLI_path_normalize_dir(job_params->main_name, dir);
-          td_dir = BLI_stack_push_r(todo_dirs);
-          td_dir->level = recursion_level + 1;
-          td_dir->dir = BLI_strdup(dir);
-          nbr_todo_dirs++;
-        }
+      if (filelist_readjob_should_recurse_into_entry(max_recursion, recursion_level, entry)) {
+        /* We have a directory we want to list, add it to todo list! */
+        BLI_join_dirfile(dir, sizeof(dir), root, entry->relpath);
+        BLI_path_normalize_dir(job_params->main_name, dir);
+        td_dir = BLI_stack_push_r(todo_dirs);
+        td_dir->level = recursion_level + 1;
+        td_dir->dir = BLI_strdup(dir);
+        nbr_todo_dirs++;
       }
     }