diff options
-rw-r--r-- | source/blender/editors/asset/CMakeLists.txt | 2 | ||||
-rw-r--r-- | source/blender/editors/asset/ED_asset_indexer.h | 50 | ||||
-rw-r--r-- | source/blender/editors/asset/intern/asset_indexer.cc | 781 | ||||
-rw-r--r-- | source/blender/editors/asset/intern/asset_list.cc | 2 | ||||
-rw-r--r-- | source/blender/editors/include/ED_file_indexer.h | 153 | ||||
-rw-r--r-- | source/blender/editors/space_file/CMakeLists.txt | 3 | ||||
-rw-r--r-- | source/blender/editors/space_file/file_indexer.cc | 96 | ||||
-rw-r--r-- | source/blender/editors/space_file/file_indexer.h | 36 | ||||
-rw-r--r-- | source/blender/editors/space_file/filelist.c | 198 | ||||
-rw-r--r-- | source/blender/editors/space_file/filelist.h | 2 | ||||
-rw-r--r-- | source/blender/editors/space_file/space_file.c | 6 |
11 files changed, 1298 insertions, 31 deletions
diff --git a/source/blender/editors/asset/CMakeLists.txt b/source/blender/editors/asset/CMakeLists.txt index b9ef8e82bba..2391f4af14d 100644 --- a/source/blender/editors/asset/CMakeLists.txt +++ b/source/blender/editors/asset/CMakeLists.txt @@ -24,6 +24,7 @@ set(INC ../../makesdna ../../makesrna ../../windowmanager + ../../../../intern/clog ../../../../intern/guardedalloc ) @@ -34,6 +35,7 @@ set(SRC intern/asset_catalog.cc intern/asset_filter.cc intern/asset_handle.cc + intern/asset_indexer.cc intern/asset_library_reference.cc intern/asset_library_reference_enum.cc intern/asset_list.cc diff --git a/source/blender/editors/asset/ED_asset_indexer.h b/source/blender/editors/asset/ED_asset_indexer.h new file mode 100644 index 00000000000..33558d8cda5 --- /dev/null +++ b/source/blender/editors/asset/ED_asset_indexer.h @@ -0,0 +1,50 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup edasset + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include "ED_file_indexer.h" + +/** + * File Indexer Service for indexing asset files. + * + * Opening and parsing a large collection of asset files inside a library can take a lot of time. + * To reduce the time it takes the files are indexed. + * + * - Index files are created for each blend file in the asset library, even when the blend file + * doesn't contain any assets. + * - Indexes are stored in an persistent cache folder (`BKE_appdir_folder_caches` + + * `asset_library_indexes/{asset_library_dir}/{asset_index_file.json}`). + * - The content of the indexes are used when: + * - Index exists and can be opened + * - Last modification date is earlier than the file it represents. + * - The index file version is the latest. + * - Blend files without any assets can be determined by the size of the index file for some + * additional performance. + */ +extern const FileIndexerType file_indexer_asset; + +#ifdef __cplusplus +} +#endif diff --git a/source/blender/editors/asset/intern/asset_indexer.cc b/source/blender/editors/asset/intern/asset_indexer.cc new file mode 100644 index 00000000000..52b5df14c9f --- /dev/null +++ b/source/blender/editors/asset/intern/asset_indexer.cc @@ -0,0 +1,781 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup edasset + */ + +#include <fstream> +#include <iomanip> +#include <optional> + +#include "ED_asset_indexer.h" + +#include "DNA_asset_types.h" +#include "DNA_userdef_types.h" + +#include "BLI_fileops.h" +#include "BLI_hash.hh" +#include "BLI_linklist.h" +#include "BLI_path_util.h" +#include "BLI_serialize.hh" +#include "BLI_set.hh" +#include "BLI_string_ref.hh" +#include "BLI_uuid.h" + +#include "BKE_appdir.h" +#include "BKE_asset.h" +#include "BKE_asset_catalog.hh" +#include "BKE_preferences.h" + +#include "CLG_log.h" + +static CLG_LogRef LOG = {"ed.asset"}; + +namespace blender::ed::asset::index { + +using namespace blender::io::serialize; +using namespace blender::bke; + +/** + * \file asset_indexer.cc + * \brief Indexer for asset libraries. + * + * Indexes are stored per input file. Each index can contain zero to multiple asset entries. + * The indexes are grouped together per asset library. They are stored in + * #BKE_appdir_folder_caches + + * /asset-library-indices/<asset-library-hash>/<asset-index-hash>_<asset_file>.index.json. + * + * The structure of an index file is + * \code + * { + * "version": <file version number>, + * "entries": [{ + * "name": "<asset name>", + * "catalog_id": "<catalog_id>", + * "catalog_name": "<catalog_name>", + * "description": "<description>", + * "author": "<author>", + * "tags": ["<tag>"] + * }] + * } + * \endcode + * + * NOTE: entries, author, description and tags are optional attributes. + * + * NOTE: File browser uses name and idcode separate. Inside the index they are joined together like + * #ID.name. + * NOTE: File browser group name isn't stored in the index as it is a translatable name. + */ +constexpr StringRef ATTRIBUTE_VERSION("version"); +constexpr StringRef ATTRIBUTE_ENTRIES("entries"); +constexpr StringRef ATTRIBUTE_ENTRIES_NAME("name"); +constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_ID("catalog_id"); +constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_NAME("catalog_name"); +constexpr StringRef ATTRIBUTE_ENTRIES_DESCRIPTION("description"); +constexpr StringRef ATTRIBUTE_ENTRIES_AUTHOR("author"); +constexpr StringRef ATTRIBUTE_ENTRIES_TAGS("tags"); + +/** Abstract class for #BlendFile and #AssetIndexFile. */ +class AbstractFile { + public: + virtual ~AbstractFile() = default; + + virtual const char *get_file_path() const = 0; + + bool exists() const + { + return BLI_exists(get_file_path()); + } + + size_t get_file_size() const + { + return BLI_file_size(get_file_path()); + } +}; + +/** + * \brief Reference to a blend file that can be indexed. + */ +class BlendFile : public AbstractFile { + StringRefNull file_path_; + + public: + BlendFile(StringRefNull file_path) : file_path_(file_path) + { + } + + uint64_t hash() const + { + DefaultHash<StringRefNull> hasher; + return hasher(file_path_); + } + + std::string get_filename() const + { + char filename[FILE_MAX]; + BLI_split_file_part(get_file_path(), filename, sizeof(filename)); + return std::string(filename); + } + + const char *get_file_path() const override + { + return file_path_.c_str(); + } +}; + +/** + * \brief Single entry inside a #AssetIndexFile for reading. + */ +struct AssetEntryReader { + private: + /** + * \brief Lookup table containing the elements of the entry. + */ + ObjectValue::Lookup lookup; + + StringRefNull get_name_with_idcode() const + { + return lookup.lookup(ATTRIBUTE_ENTRIES_NAME)->as_string_value()->value(); + } + + public: + AssetEntryReader(const ObjectValue &entry) : lookup(entry.create_lookup()) + { + } + + ID_Type get_idcode() const + { + const StringRefNull name_with_idcode = get_name_with_idcode(); + return GS(name_with_idcode.c_str()); + } + + StringRef get_name() const + { + const StringRefNull name_with_idcode = get_name_with_idcode(); + return name_with_idcode.substr(2); + } + + bool has_description() const + { + return lookup.contains(ATTRIBUTE_ENTRIES_DESCRIPTION); + } + + StringRefNull get_description() const + { + return lookup.lookup(ATTRIBUTE_ENTRIES_DESCRIPTION)->as_string_value()->value(); + } + + bool has_author() const + { + return lookup.contains(ATTRIBUTE_ENTRIES_AUTHOR); + } + + StringRefNull get_author() const + { + return lookup.lookup(ATTRIBUTE_ENTRIES_AUTHOR)->as_string_value()->value(); + } + + StringRefNull get_catalog_name() const + { + return lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_NAME)->as_string_value()->value(); + } + + CatalogID get_catalog_id() const + { + const std::string &catalog_id = + lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_ID)->as_string_value()->value(); + CatalogID catalog_uuid(catalog_id); + return catalog_uuid; + } + + void add_tags_to_meta_data(AssetMetaData *asset_data) const + { + const ObjectValue::LookupValue *value_ptr = lookup.lookup_ptr(ATTRIBUTE_ENTRIES_TAGS); + if (value_ptr == nullptr) { + return; + } + + const ArrayValue *array_value = (*value_ptr)->as_array_value(); + const ArrayValue::Items &elements = array_value->elements(); + for (const ArrayValue::Item &item : elements) { + const StringRefNull tag_name = item->as_string_value()->value(); + BKE_asset_metadata_tag_add(asset_data, tag_name.c_str()); + } + } +}; + +struct AssetEntryWriter { + private: + ObjectValue::Items &attributes; + + public: + AssetEntryWriter(ObjectValue &entry) : attributes(entry.elements()) + { + } + + /** + * \brief add id + name to the attributes. + * + * NOTE: id and name are encoded like #ID.name + */ + void add_id_name(const short idcode, const StringRefNull name) + { + char idcode_prefix[2]; + /* Similar to `BKE_libblock_alloc`. */ + *((short *)idcode_prefix) = idcode; + std::string name_with_idcode = std::string(idcode_prefix) + name; + + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_NAME, new StringValue(name_with_idcode))); + } + + void add_catalog_id(const CatalogID &catalog_id) + { + char catalog_id_str[UUID_STRING_LEN]; + BLI_uuid_format(catalog_id_str, catalog_id); + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_ID, new StringValue(catalog_id_str))); + } + + void add_catalog_name(const StringRefNull catalog_name) + { + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_NAME, new StringValue(catalog_name))); + } + + void add_description(const StringRefNull description) + { + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_DESCRIPTION, new StringValue(description))); + } + + void add_author(const StringRefNull author) + { + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_AUTHOR, new StringValue(author))); + } + + void add_tags(const ListBase /* AssetTag */ *asset_tags) + { + ArrayValue *tags = new ArrayValue(); + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_TAGS, tags)); + ArrayValue::Items &tag_items = tags->elements(); + + LISTBASE_FOREACH (AssetTag *, tag, asset_tags) { + tag_items.append_as(new StringValue(tag->name)); + } + } +}; + +static void init_value_from_file_indexer_entry(AssetEntryWriter &result, + const FileIndexerEntry *indexer_entry) +{ + const BLODataBlockInfo &datablock_info = indexer_entry->datablock_info; + + result.add_id_name(indexer_entry->idcode, datablock_info.name); + + const AssetMetaData &asset_data = *datablock_info.asset_data; + result.add_catalog_id(asset_data.catalog_id); + result.add_catalog_name(asset_data.catalog_simple_name); + + if (asset_data.description != nullptr) { + result.add_description(asset_data.description); + } + if (asset_data.author != nullptr) { + result.add_author(asset_data.author); + } + + if (!BLI_listbase_is_empty(&asset_data.tags)) { + result.add_tags(&asset_data.tags); + } + + /* TODO: asset_data.IDProperties */ +} + +static void init_value_from_file_indexer_entries(ObjectValue &result, + const FileIndexerEntries &indexer_entries) +{ + ArrayValue *entries = new ArrayValue(); + ArrayValue::Items &items = entries->elements(); + + for (LinkNode *ln = indexer_entries.entries; ln; ln = ln->next) { + const FileIndexerEntry *indexer_entry = static_cast<const FileIndexerEntry *>(ln->link); + /* We also get non asset types (brushes, workspaces), when browsing using the asset browser. */ + if (indexer_entry->datablock_info.asset_data == nullptr) { + continue; + } + ObjectValue *entry_value = new ObjectValue(); + AssetEntryWriter entry(*entry_value); + init_value_from_file_indexer_entry(entry, indexer_entry); + items.append_as(entry_value); + } + + /* When no entries to index, we should not store the entries attribute as this would make the + * size bigger than the #MIN_FILE_SIZE_WITH_ENTRIES. */ + if (items.is_empty()) { + delete entries; + return; + } + + ObjectValue::Items &attributes = result.elements(); + attributes.append_as(std::pair(ATTRIBUTE_ENTRIES, entries)); +} + +static void init_indexer_entry_from_value(FileIndexerEntry &indexer_entry, + const AssetEntryReader &entry) +{ + indexer_entry.idcode = entry.get_idcode(); + + const std::string &name = entry.get_name(); + BLI_strncpy( + indexer_entry.datablock_info.name, name.c_str(), sizeof(indexer_entry.datablock_info.name)); + + AssetMetaData *asset_data = BKE_asset_metadata_create(); + indexer_entry.datablock_info.asset_data = asset_data; + + if (entry.has_description()) { + const std::string &description = entry.get_description(); + char *description_c_str = static_cast<char *>(MEM_mallocN(description.length() + 1, __func__)); + BLI_strncpy(description_c_str, description.c_str(), description.length() + 1); + asset_data->description = description_c_str; + } + if (entry.has_author()) { + const std::string &author = entry.get_author(); + char *author_c_str = static_cast<char *>(MEM_mallocN(author.length() + 1, __func__)); + BLI_strncpy(author_c_str, author.c_str(), author.length() + 1); + asset_data->author = author_c_str; + } + + const std::string &catalog_name = entry.get_catalog_name(); + BLI_strncpy(asset_data->catalog_simple_name, + catalog_name.c_str(), + sizeof(asset_data->catalog_simple_name)); + + asset_data->catalog_id = entry.get_catalog_id(); + + entry.add_tags_to_meta_data(asset_data); +} + +static int init_indexer_entries_from_value(FileIndexerEntries &indexer_entries, + const ObjectValue &value) +{ + const ObjectValue::Lookup attributes = value.create_lookup(); + const ObjectValue::LookupValue *entries_value = attributes.lookup_ptr(ATTRIBUTE_ENTRIES); + BLI_assert(entries_value != nullptr); + + if (entries_value == nullptr) { + return 0; + } + + int num_entries_read = 0; + const ArrayValue::Items elements = (*entries_value)->as_array_value()->elements(); + for (ArrayValue::Item element : elements) { + const AssetEntryReader asset_entry(*element->as_object_value()); + + FileIndexerEntry *entry = static_cast<FileIndexerEntry *>( + MEM_callocN(sizeof(FileIndexerEntry), __func__)); + init_indexer_entry_from_value(*entry, asset_entry); + + BLI_linklist_prepend(&indexer_entries.entries, entry); + num_entries_read += 1; + } + + return num_entries_read; +} + +/** + * \brief References the asset library directory. + * + * The #AssetLibraryIndex instance is used to keep track of unused file indices. When reading any + * used indices are removed from the list and when reading is finished the unused + * indices are removed. + */ +struct AssetLibraryIndex { + /** + * Tracks indices that haven't been used yet. + * + * Contains absolute paths to the indices. + */ + Set<std::string> unused_file_indices; + + /** + * \brief Absolute path where the indices of `library` are stored. + * + * \NOTE: includes trailing directory separator. + */ + std::string indices_base_path; + + std::string library_path; + + public: + AssetLibraryIndex(const StringRef library_path) : library_path(library_path) + { + init_indices_base_path(); + } + + uint64_t hash() const + { + DefaultHash<StringRefNull> hasher; + return hasher(get_library_file_path()); + } + + StringRefNull get_library_file_path() const + { + return library_path; + } + + /** + * \brief Initializes #AssetLibraryIndex.indices_base_path. + * + * `BKE_appdir_folder_caches/asset-library-indices/<asset-library-name-hash>/` + */ + void init_indices_base_path() + { + char index_path[FILE_MAX]; + BKE_appdir_folder_caches(index_path, sizeof(index_path)); + + BLI_path_append(index_path, sizeof(index_path), "asset-library-indices"); + + std::stringstream ss; + ss << std::setfill('0') << std::setw(16) << std::hex << hash() << "/"; + BLI_path_append(index_path, sizeof(index_path), ss.str().c_str()); + + indices_base_path = std::string(index_path); + } + + /** + * \return absolute path to the index file of the given `asset_file`. + * + * `{indices_base_path}/{asset-file_hash}_{asset-file-filename}.index.json`. + */ + std::string index_file_path(const BlendFile &asset_file) const + { + std::stringstream ss; + ss << indices_base_path; + ss << std::setfill('0') << std::setw(16) << std::hex << asset_file.hash() << "_" + << asset_file.get_filename() << ".index.json"; + return ss.str(); + } + + /** + * Initialize to keep track of unused file indices. + */ + void init_unused_index_files() + { + const char *index_path = indices_base_path.c_str(); + if (!BLI_is_dir(index_path)) { + return; + } + struct direntry *dir_entries = nullptr; + int num_entries = BLI_filelist_dir_contents(index_path, &dir_entries); + for (int i = 0; i < num_entries; i++) { + struct direntry *entry = &dir_entries[i]; + if (BLI_str_endswith(entry->relname, ".index.json")) { + unused_file_indices.add_as(std::string(entry->path)); + } + } + + BLI_filelist_free(dir_entries, num_entries); + } + + void mark_as_used(const std::string &filename) + { + unused_file_indices.remove(filename); + } + + int remove_unused_index_files() const + { + int num_files_deleted = 0; + for (const std::string &unused_index : unused_file_indices) { + const char *file_path = unused_index.c_str(); + CLOG_INFO(&LOG, 2, "Remove unused index file [%s].", file_path); + BLI_delete(file_path, false, false); + num_files_deleted++; + } + + return num_files_deleted; + } +}; + +/** + * Instance of this class represents the contents of an asset index file. + * + * /code + * { + * "version": {version}, + * "entries": ... + * } + * /endcode + */ +struct AssetIndex { + /** + * \brief Version to store in new index files. + * + * Versions are written to each index file. When reading the version is checked against + * `CURRENT_VERSION` to make sure we can use the index. Developer should increase + * `CURRENT_VERSION` when changes are made to the structure of the stored index. + */ + static const int CURRENT_VERSION = 1; + + /** + * Version number to use when version couldn't be read from an index file. + */ + const int UNKNOWN_VERSION = -1; + + /** + * `blender::io::serialize::Value` represeting the contents of an index file. + * + * Value is used over ObjectValue as the contents of the index could be corrupted and doesn't + * represent an object. In case corrupted files are detected the `get_version` would return + * UNKNOWN_VERSION.` + */ + std::unique_ptr<Value> contents; + + /** + * Constructor for when creating/updating an asset index file. + * #AssetIndex.contents are filled from the given \p indexer_entries. + */ + AssetIndex(const FileIndexerEntries &indexer_entries) + { + std::unique_ptr<ObjectValue> root = std::make_unique<ObjectValue>(); + ObjectValue::Items &root_attributes = root->elements(); + root_attributes.append_as(std::pair(ATTRIBUTE_VERSION, new IntValue(CURRENT_VERSION))); + init_value_from_file_indexer_entries(*root, indexer_entries); + + contents = std::move(root); + } + + /** + * Constructor when reading an asset index file. + * #AssetIndex.contents are read from the given \p value. + */ + AssetIndex(std::unique_ptr<Value> &value) : contents(std::move(value)) + { + } + + int get_version() const + { + const ObjectValue *root = contents->as_object_value(); + if (root == nullptr) { + return UNKNOWN_VERSION; + } + const ObjectValue::Lookup attributes = root->create_lookup(); + const ObjectValue::LookupValue *version_value = attributes.lookup_ptr(ATTRIBUTE_VERSION); + if (version_value == nullptr) { + return UNKNOWN_VERSION; + } + return (*version_value)->as_int_value()->value(); + } + + bool is_latest_version() const + { + return get_version() == CURRENT_VERSION; + } + + /** + * Extract the contents of this index into the given \p indexer_entries. + * + * \return The number of entries read from the given entries. + */ + int extract_into(FileIndexerEntries &indexer_entries) const + { + const ObjectValue *root = contents->as_object_value(); + const int num_entries_read = init_indexer_entries_from_value(indexer_entries, *root); + return num_entries_read; + } +}; + +class AssetIndexFile : public AbstractFile { + public: + AssetLibraryIndex &library_index; + /** + * Asset index files with a size smaller than this attribute would be considered to not contain + * any entries. + */ + const size_t MIN_FILE_SIZE_WITH_ENTRIES = 32; + std::string filename; + + AssetIndexFile(AssetLibraryIndex &library_index, BlendFile &asset_filename) + : library_index(library_index), filename(library_index.index_file_path(asset_filename)) + { + } + + void mark_as_used() + { + library_index.mark_as_used(filename); + } + + const char *get_file_path() const override + { + return filename.c_str(); + } + + /** + * Returns whether the index file is older than the given asset file. + */ + bool is_older_than(BlendFile &asset_file) const + { + return BLI_file_older(get_file_path(), asset_file.get_file_path()); + } + + /** + * Check whether the index file contains entries without opening the file. + */ + bool constains_entries() const + { + const size_t file_size = get_file_size(); + return file_size >= MIN_FILE_SIZE_WITH_ENTRIES; + } + + std::unique_ptr<AssetIndex> read_contents() const + { + JsonFormatter formatter; + std::ifstream is; + is.open(filename); + std::unique_ptr<Value> read_data = formatter.deserialize(is); + is.close(); + + return std::make_unique<AssetIndex>(read_data); + } + + bool ensure_parent_path_exists() const + { + /* `BLI_make_existing_file` only ensures parent path, otherwise than expected from the name of + * the function. */ + return BLI_make_existing_file(get_file_path()); + } + + void write_contents(AssetIndex &content) + { + JsonFormatter formatter; + if (!ensure_parent_path_exists()) { + CLOG_ERROR(&LOG, "Index not created: couldn't create folder [%s].", get_file_path()); + return; + } + + std::ofstream os; + os.open(filename, std::ios::out | std::ios::trunc); + formatter.serialize(os, *content.contents); + os.close(); + } +}; + +static eFileIndexerResult read_index(const char *filename, + FileIndexerEntries *entries, + int *r_read_entries_len, + void *user_data) +{ + AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data); + BlendFile asset_file(filename); + AssetIndexFile asset_index_file(library_index, asset_file); + + if (!asset_index_file.exists()) { + return FILE_INDEXER_NEEDS_UPDATE; + } + + /* Mark index as used, even when it will be recreated. When not done it would remove the index + * when the indexing has finished (see `AssetLibraryIndex.remove_unused_index_files`), thereby + * removing the newly created index. + */ + asset_index_file.mark_as_used(); + + if (asset_index_file.is_older_than(asset_file)) { + CLOG_INFO( + &LOG, + 3, + "Asset index file [%s] needs to be refreshed as it is older than the asset file [%s].", + asset_index_file.filename.c_str(), + filename); + return FILE_INDEXER_NEEDS_UPDATE; + } + + if (!asset_index_file.constains_entries()) { + CLOG_INFO(&LOG, + 3, + "Asset file index is to small to contain any entries. [%s]", + asset_index_file.filename.c_str()); + *r_read_entries_len = 0; + return FILE_INDEXER_ENTRIES_LOADED; + } + + std::unique_ptr<AssetIndex> contents = asset_index_file.read_contents(); + if (!contents->is_latest_version()) { + CLOG_INFO(&LOG, + 3, + "Asset file index is ignored; expected version %d but file is version %d [%s].", + AssetIndex::CURRENT_VERSION, + contents->get_version(), + asset_index_file.filename.c_str()); + return FILE_INDEXER_NEEDS_UPDATE; + } + + const int read_entries_len = contents->extract_into(*entries); + CLOG_INFO(&LOG, 1, "Read %d entries from asset index for [%s].", read_entries_len, filename); + *r_read_entries_len = read_entries_len; + + return FILE_INDEXER_ENTRIES_LOADED; +} + +static void update_index(const char *filename, FileIndexerEntries *entries, void *user_data) +{ + AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data); + BlendFile asset_file(filename); + AssetIndexFile asset_index_file(library_index, asset_file); + CLOG_INFO(&LOG, + 1, + "Update asset index for [%s] store index in [%s].", + asset_file.get_file_path(), + asset_index_file.get_file_path()); + + AssetIndex content(*entries); + asset_index_file.write_contents(content); +} + +static void *init_user_data(const char *root_directory, size_t root_directory_maxlen) +{ + AssetLibraryIndex *library_index = OBJECT_GUARDED_NEW( + AssetLibraryIndex, + StringRef(root_directory, BLI_strnlen(root_directory, root_directory_maxlen))); + library_index->init_unused_index_files(); + return library_index; +} + +static void free_user_data(void *user_data) +{ + OBJECT_GUARDED_DELETE(user_data, AssetLibraryIndex); +} + +static void filelist_finished(void *user_data) +{ + AssetLibraryIndex &library_index = *static_cast<AssetLibraryIndex *>(user_data); + const int num_indices_removed = library_index.remove_unused_index_files(); + if (num_indices_removed > 0) { + CLOG_INFO(&LOG, 1, "Removed %d unused indices.", num_indices_removed); + } +} + +constexpr FileIndexerType asset_indexer() +{ + FileIndexerType indexer = {nullptr}; + indexer.read_index = read_index; + indexer.update_index = update_index; + indexer.init_user_data = init_user_data; + indexer.free_user_data = free_user_data; + indexer.filelist_finished = filelist_finished; + return indexer; +} + +} // namespace blender::ed::asset::index + +extern "C" { +const FileIndexerType file_indexer_asset = blender::ed::asset::index::asset_indexer(); +} diff --git a/source/blender/editors/asset/intern/asset_list.cc b/source/blender/editors/asset/intern/asset_list.cc index c1b1e33d428..6079e91d5f5 100644 --- a/source/blender/editors/asset/intern/asset_list.cc +++ b/source/blender/editors/asset/intern/asset_list.cc @@ -44,6 +44,7 @@ #include "../space_file/filelist.h" #include "ED_asset_handle.h" +#include "ED_asset_indexer.h" #include "ED_asset_list.h" #include "ED_asset_list.hh" #include "asset_library_reference.hh" @@ -168,6 +169,7 @@ void AssetList::setup() true, "", ""); + filelist_setindexer(files, &file_indexer_asset); char path[FILE_MAXDIR] = ""; if (user_library) { diff --git a/source/blender/editors/include/ED_file_indexer.h b/source/blender/editors/include/ED_file_indexer.h new file mode 100644 index 00000000000..12579283a62 --- /dev/null +++ b/source/blender/editors/include/ED_file_indexer.h @@ -0,0 +1,153 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup edfile + */ + +#pragma once + +#include "BLO_readfile.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * File indexing for the file/asset browser. + * + * This file contains an API to create indexing functionality when listing blend files in + * the file browser. + * + * To implement a custom indexer a `FileIndexerType` struct should be made and passed to the + * `filelist_setindexer` function. + */ + +struct AssetLibraryReference; +struct LinkNode; + +/** + * Result code of the `read_index` callback. + */ +typedef enum eFileIndexerResult { + /** + * File listing entries are loaded from the index. Reading entries from the blend file itself + * should be skipped. + */ + FILE_INDEXER_ENTRIES_LOADED, + + /** + * Index isn't available or not upto date. Entries should be read from te blend file and + * `update_index` must be called to update the index. + */ + FILE_INDEXER_NEEDS_UPDATE, +} eFileIndexerResult; + +/** + * FileIndexerEntry contains all data that is required to create a file listing entry. + */ +typedef struct FileIndexerEntry { + struct BLODataBlockInfo datablock_info; + short idcode; +} FileIndexerEntry; + +/** + * Contains all entries of a blend file. + */ +typedef struct FileIndexerEntries { + struct LinkNode /* FileIndexerEntry */ *entries; +} FileIndexerEntries; + +typedef void *(*FileIndexerInitUserDataFunc)(const char *root_directory, + size_t root_directory_maxlen); +typedef void (*FileIndexerFreeUserDataFunc)(void *); +typedef void (*FileIndexerFinishedFunc)(void *); +typedef eFileIndexerResult (*FileIndexerReadIndexFunc)(const char *file_name, + FileIndexerEntries *entries, + int *r_read_entries_len, + void *user_data); +typedef void (*FileIndexerUpdateIndexFunc)(const char *file_name, + FileIndexerEntries *entries, + void *user_data); + +typedef struct FileIndexerType { + /** + * Is called at the beginning of the file listing process. An indexer can + * setup needed data. The result of this function will be passed around as `user_data` parameter. + * + * This is an optional callback. + */ + FileIndexerInitUserDataFunc init_user_data; + + /** + * Is called at the end of the file listing process. An indexer can free the data that it created + * during the file listing process. + * + * This is an optional callback */ + FileIndexerFreeUserDataFunc free_user_data; + + /** + * Is called at the end of the file listing process (before the `free_user_data`) where indexes + * can perform clean-ups. + * + * This is an optinal callback. Called when listing files completed. + */ + FileIndexerFinishedFunc filelist_finished; + + /** + * Is called for each blend file being listed to read data from the index. + * + * Read entries should be added to given `entries` parameter (type: `FileIndexerEntries`). + * `*r_read_entries_len` must be set to the number of read entries. + * and the function must return `eFileIndexerResult::FILE_INDEXER_ENTRIES_LOADED`. + * In this case the blend file will not be opened and the FileIndexerEntry added to `entries` + * will be used as the content of the file. + * + * When the index isn't available or could not be used no entries must be added to the + * entries field, `r_read_entries_len` must be set to `0` and the function must return + * `eFileIndexerResult::FILE_INDEXER_NEEDS_UPDATE`. In this case the blend file will read from + * the blend file and the `update_index` function will be called. + */ + FileIndexerReadIndexFunc read_index; + + /** + * Update an index of a blend file. + * + * Is called after reading entries from the file when the result of `read_index` was + * `eFileIndexerResult::FILE_INDEXER_NEED_UPDATE`. The callback should update the index so the + * next time that read_index is called it will read the entries from the index. + */ + FileIndexerUpdateIndexFunc update_index; +} FileIndexerType; + +/* file_indexer.cc */ + +/** Removes all entries inside the given `indexer_entries`. */ +void ED_file_indexer_entries_clear(FileIndexerEntries *indexer_entries); + +/** + * Adds all entries from the given `datablock_infos` to the `indexer_entries`. + * The datablock_infos must only contain data for a single IDType. The specific IDType must be + * passed in the `idcode` parameter. + */ +void ED_file_indexer_entries_extend_from_datablock_infos( + FileIndexerEntries *indexer_entries, + const LinkNode * /* BLODataBlockInfo */ datablock_infos, + const int idcode); + +#ifdef __cplusplus +} +#endif diff --git a/source/blender/editors/space_file/CMakeLists.txt b/source/blender/editors/space_file/CMakeLists.txt index 4b508f16c1e..1651269869e 100644 --- a/source/blender/editors/space_file/CMakeLists.txt +++ b/source/blender/editors/space_file/CMakeLists.txt @@ -17,6 +17,7 @@ set(INC ../include + ../asset ../../blenfont ../../blenkernel ../../blenlib @@ -36,6 +37,7 @@ set(INC set(SRC asset_catalog_tree_view.cc file_draw.c + file_indexer.cc file_ops.c file_panels.c file_utils.c @@ -44,6 +46,7 @@ set(SRC fsmenu.c space_file.c + file_indexer.h file_intern.h filelist.h fsmenu.h diff --git a/source/blender/editors/space_file/file_indexer.cc b/source/blender/editors/space_file/file_indexer.cc new file mode 100644 index 00000000000..95e0afd7a1e --- /dev/null +++ b/source/blender/editors/space_file/file_indexer.cc @@ -0,0 +1,96 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup edfile + * + * This file implements the default file browser indexer and has some helper function to work with + * `FileIndexerEntries`. + */ +#include "file_indexer.h" + +#include "MEM_guardedalloc.h" + +#include "BLI_linklist.h" +#include "BLI_listbase.h" +#include "BLI_string.h" +#include "BLI_utildefines.h" + +namespace blender::ed::file::indexer { + +static eFileIndexerResult read_index(const char *UNUSED(file_name), + FileIndexerEntries *UNUSED(entries), + int *UNUSED(r_read_entries_len), + void *UNUSED(user_data)) +{ + return FILE_INDEXER_NEEDS_UPDATE; +} + +static void update_index(const char *UNUSED(file_name), + FileIndexerEntries *UNUSED(entries), + void *UNUSED(user_data)) +{ +} + +constexpr FileIndexerType default_indexer() +{ + FileIndexerType indexer = {nullptr}; + indexer.read_index = read_index; + indexer.update_index = update_index; + return indexer; +} + +static FileIndexerEntry *file_indexer_entry_create_from_datablock_info( + const BLODataBlockInfo *datablock_info, const int idcode) +{ + FileIndexerEntry *entry = static_cast<FileIndexerEntry *>( + MEM_mallocN(sizeof(FileIndexerEntry), __func__)); + entry->datablock_info = *datablock_info; + entry->idcode = idcode; + return entry; +} + +} // namespace blender::ed::file::indexer + +extern "C" { + +void ED_file_indexer_entries_extend_from_datablock_infos( + FileIndexerEntries *indexer_entries, + const LinkNode * /* BLODataBlockInfo */ datablock_infos, + const int idcode) +{ + for (const LinkNode *ln = datablock_infos; ln; ln = ln->next) { + const BLODataBlockInfo *datablock_info = static_cast<const BLODataBlockInfo *>(ln->link); + FileIndexerEntry *file_indexer_entry = + blender::ed::file::indexer::file_indexer_entry_create_from_datablock_info(datablock_info, + idcode); + BLI_linklist_prepend(&indexer_entries->entries, file_indexer_entry); + } +} + +static void ED_file_indexer_entry_free(void *indexer_entry) +{ + MEM_freeN(indexer_entry); +} + +void ED_file_indexer_entries_clear(FileIndexerEntries *indexer_entries) +{ + BLI_linklist_free(indexer_entries->entries, ED_file_indexer_entry_free); + indexer_entries->entries = nullptr; +} + +const FileIndexerType file_indexer_noop = blender::ed::file::indexer::default_indexer(); +} diff --git a/source/blender/editors/space_file/file_indexer.h b/source/blender/editors/space_file/file_indexer.h new file mode 100644 index 00000000000..ea42f10498b --- /dev/null +++ b/source/blender/editors/space_file/file_indexer.h @@ -0,0 +1,36 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup edfile + */ +#pragma once + +#include "ED_file_indexer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Default indexer to use when listing files. The implementation is a no-operation indexing. When + * set it won't use indexing. It is added to increase the code clarity. + */ +extern const FileIndexerType file_indexer_noop; + +#ifdef __cplusplus +} +#endif diff --git a/source/blender/editors/space_file/filelist.c b/source/blender/editors/space_file/filelist.c index 045ebd10c0a..d4e9118960a 100644 --- a/source/blender/editors/space_file/filelist.c +++ b/source/blender/editors/space_file/filelist.c @@ -89,6 +89,7 @@ #include "atomic_ops.h" +#include "file_indexer.h" #include "file_intern.h" #include "filelist.h" @@ -396,6 +397,11 @@ typedef struct FileList { FileListFilter filter_data; + /** + * File indexer to use. Attribute is always set. + */ + const struct FileIndexerType *indexer; + struct FileListIntern filelist_intern; struct FileListEntryCache filelist_cache; @@ -1126,6 +1132,19 @@ void filelist_setfilter_options(FileList *filelist, } /** + * Set the indexer to be used by the filelist. + * + * The given indexer allocation should be handled by the caller or defined statically. + */ +void filelist_setindexer(FileList *filelist, const FileIndexerType *indexer) +{ + BLI_assert(filelist); + BLI_assert(indexer); + + filelist->indexer = indexer; +} + +/** * \param catalog_id: The catalog that should be filtered by if \a catalog_visibility is * #FILE_SHOW_ASSETS_FROM_CATALOG. May be NULL otherwise. */ @@ -1830,6 +1849,8 @@ FileList *filelist_new(short type) p->filelist.nbr_entries = FILEDIR_NBR_ENTRIES_UNSET; filelist_settype(p, type); + p->indexer = &file_indexer_noop; + return p; } @@ -3120,6 +3141,29 @@ static FileListInternEntry *filelist_readjob_list_lib_group_create(const int idc return entry; } +static void filelist_readjob_list_lib_add_datablock(ListBase *entries, + const BLODataBlockInfo *datablock_info, + const bool prefix_relpath_with_group_name, + const int idcode, + const char *group_name) +{ + FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__); + if (prefix_relpath_with_group_name) { + entry->relpath = BLI_sprintfN("%s/%s", group_name, datablock_info->name); + } + else { + entry->relpath = BLI_strdup(datablock_info->name); + } + entry->typeflag |= FILE_TYPE_BLENDERLIB; + if (datablock_info && datablock_info->asset_data) { + entry->typeflag |= FILE_TYPE_ASSET; + /* Moves ownership! */ + entry->imported_asset_data = datablock_info->asset_data; + } + entry->blentype = idcode; + BLI_addtail(entries, entry); +} + static void filelist_readjob_list_lib_add_datablocks(ListBase *entries, LinkNode *datablock_infos, const bool prefix_relpath_with_group_name, @@ -3127,29 +3171,71 @@ static void filelist_readjob_list_lib_add_datablocks(ListBase *entries, const char *group_name) { for (LinkNode *ln = datablock_infos; ln; ln = ln->next) { - struct BLODataBlockInfo *info = ln->link; - FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__); - if (prefix_relpath_with_group_name) { - entry->relpath = BLI_sprintfN("%s/%s", group_name, info->name); - } - else { - entry->relpath = BLI_strdup(info->name); - } - entry->typeflag |= FILE_TYPE_BLENDERLIB; - if (info && info->asset_data) { - entry->typeflag |= FILE_TYPE_ASSET; - /* Moves ownership! */ - entry->imported_asset_data = info->asset_data; - } - entry->blentype = idcode; + struct BLODataBlockInfo *datablock_info = ln->link; + filelist_readjob_list_lib_add_datablock( + entries, datablock_info, prefix_relpath_with_group_name, idcode, group_name); + } +} + +static void filelist_readjob_list_lib_add_from_indexer_entries( + ListBase *entries, + const FileIndexerEntries *indexer_entries, + const bool prefix_relpath_with_group_name) +{ + for (const LinkNode *ln = indexer_entries->entries; ln; ln = ln->next) { + const FileIndexerEntry *indexer_entry = (const FileIndexerEntry *)ln->link; + const char *group_name = BKE_idtype_idcode_to_name(indexer_entry->idcode); + filelist_readjob_list_lib_add_datablock(entries, + &indexer_entry->datablock_info, + prefix_relpath_with_group_name, + indexer_entry->idcode, + group_name); + } +} + +static FileListInternEntry *filelist_readjob_list_lib_navigate_to_parent_entry_create(void) +{ + FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__); + entry->relpath = BLI_strdup(FILENAME_PARENT); + entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR); + return entry; +} + +/** + * Structure to keep the file indexer and its user data together. + */ +typedef struct FileIndexer { + const FileIndexerType *callbacks; + + /** + * User data. Contains the result of `callbacks.init_user_data`. + */ + void *user_data; +} FileIndexer; + +static int filelist_readjob_list_lib_populate_from_index(ListBase *entries, + const ListLibOptions options, + const int read_from_index, + const FileIndexerEntries *indexer_entries) +{ + int navigate_to_parent_len = 0; + if (options & LIST_LIB_ADD_PARENT) { + FileListInternEntry *entry = filelist_readjob_list_lib_navigate_to_parent_entry_create(); BLI_addtail(entries, entry); + navigate_to_parent_len = 1; } + + filelist_readjob_list_lib_add_from_indexer_entries(entries, indexer_entries, true); + return read_from_index + navigate_to_parent_len; } static int filelist_readjob_list_lib(const char *root, ListBase *entries, - const ListLibOptions options) + const ListLibOptions options, + FileIndexer *indexer_runtime) { + BLI_assert(indexer_runtime); + char dir[FILE_MAX_LIBEXTRA], *group; struct BlendHandle *libfiledata = NULL; @@ -3157,13 +3243,37 @@ static int filelist_readjob_list_lib(const char *root, /* Check if the given root is actually a library. All folders are passed to * `filelist_readjob_list_lib` and based on the number of found entries `filelist_readjob_do` * will do a dir listing only when this function does not return any entries. */ - /* TODO: We should consider introducing its own function to detect if it is a lib and + /* TODO(jbakker): We should consider introducing its own function to detect if it is a lib and * call it directly from `filelist_readjob_do` to increase readability. */ const bool is_lib = BLO_library_path_explode(root, dir, &group, NULL); if (!is_lib) { return 0; } + const bool group_came_from_path = group != NULL; + + /* Try read from indexer_runtime. */ + /* Indexing returns all entries in a blend file. We should ignore the index when listing a group + * inside a blend file, so the `entries` isn't filled with undesired entries. + * This happens when linking or appending data-blocks, where you can navigate into a group (fe + * Materials/Objects) where you only want to work with partial indexes. + * + * Adding support for partial reading/updating indexes would increase the complexity. + */ + const bool use_indexer = !group_came_from_path; + FileIndexerEntries indexer_entries = {NULL}; + if (use_indexer) { + int read_from_index = 0; + eFileIndexerResult indexer_result = indexer_runtime->callbacks->read_index( + dir, &indexer_entries, &read_from_index, indexer_runtime->user_data); + if (indexer_result == FILE_INDEXER_ENTRIES_LOADED) { + int entries_read = filelist_readjob_list_lib_populate_from_index( + entries, options, read_from_index, &indexer_entries); + ED_file_indexer_entries_clear(&indexer_entries); + return entries_read; + } + } + /* Open the library file. */ BlendFileReadReport bf_reports = {.reports = NULL}; libfiledata = BLO_blendhandle_from_file(dir, &bf_reports); @@ -3172,18 +3282,18 @@ static int filelist_readjob_list_lib(const char *root, } /* Add current parent when requested. */ - int parent_len = 0; + /* Is the navigate to previous level added to the list of entries. When added the return value + * should be increased to match the actual number of entries added. It is introduced to keep + * the code clean and readable and not counting in a single variable. */ + int navigate_to_parent_len = 0; if (options & LIST_LIB_ADD_PARENT) { - FileListInternEntry *entry = MEM_callocN(sizeof(*entry), __func__); - entry->relpath = BLI_strdup(FILENAME_PARENT); - entry->typeflag |= (FILE_TYPE_BLENDERLIB | FILE_TYPE_DIR); + FileListInternEntry *entry = filelist_readjob_list_lib_navigate_to_parent_entry_create(); BLI_addtail(entries, entry); - parent_len = 1; + navigate_to_parent_len = 1; } int group_len = 0; int datablock_len = 0; - const bool group_came_from_path = group != NULL; if (group_came_from_path) { const int idcode = groupname_to_code(group); LinkNode *datablock_infos = BLO_blendhandle_get_datablock_info( @@ -3208,6 +3318,10 @@ static int filelist_readjob_list_lib(const char *root, libfiledata, idcode, options & LIST_LIB_ASSETS_ONLY, &group_datablock_len); filelist_readjob_list_lib_add_datablocks( entries, group_datablock_infos, true, idcode, group_name); + if (use_indexer) { + ED_file_indexer_entries_extend_from_datablock_infos( + &indexer_entries, group_datablock_infos, idcode); + } BLI_linklist_freeN(group_datablock_infos); datablock_len += group_datablock_len; } @@ -3218,8 +3332,14 @@ static int filelist_readjob_list_lib(const char *root, BLO_blendhandle_close(libfiledata); + /* Update the index. */ + if (use_indexer) { + indexer_runtime->callbacks->update_index(dir, &indexer_entries, indexer_runtime->user_data); + ED_file_indexer_entries_clear(&indexer_entries); + } + /* Return the number of items added to entries. */ - int added_entries_len = group_len + datablock_len + parent_len; + int added_entries_len = group_len + datablock_len + navigate_to_parent_len; return added_entries_len; } @@ -3411,11 +3531,11 @@ typedef struct FileListReadJob { * The job system calls #filelist_readjob_update which moves any read file from #tmp_filelist * into #filelist in a thread-safe way. * - * #tmp_filelist also keeps an `AssetLibrary *` so that it can be loaded in the same thread, and - * moved to #filelist once all categories are loaded. + * #tmp_filelist also keeps an `AssetLibrary *` so that it can be loaded in the same thread, + * and moved to #filelist once all categories are loaded. * - * NOTE: #tmp_filelist is freed in #filelist_readjob_free, so any copied pointers need to be set - * to NULL to avoid double-freeing them. */ + * NOTE: #tmp_filelist is freed in #filelist_readjob_free, so any copied pointers need to be + * set to NULL to avoid double-freeing them. */ struct FileList *tmp_filelist; } FileListReadJob; @@ -3452,8 +3572,8 @@ static bool filelist_readjob_should_recurse_into_entry(const int max_recursion, /* No more levels of recursion left. */ return false; } - /* Show entries when recursion is set to `Blend file` even when `current_recursion_level` exceeds - * `max_recursion`. */ + /* Show entries when recursion is set to `Blend file` even when `current_recursion_level` + * exceeds `max_recursion`. */ if (!is_lib && (current_recursion_level >= max_recursion) && ((entry->typeflag & (FILE_TYPE_BLENDER | FILE_TYPE_BLENDER_BACKUP)) == 0)) { return false; @@ -3501,6 +3621,12 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib, BLI_path_normalize_dir(job_params->main_name, dir); td_dir->dir = BLI_strdup(dir); + /* Init the file indexer. */ + FileIndexer indexer_runtime = {.callbacks = filelist->indexer}; + if (indexer_runtime.callbacks->init_user_data) { + indexer_runtime.user_data = indexer_runtime.callbacks->init_user_data(dir, sizeof(dir)); + } + while (!BLI_stack_is_empty(todo_dirs) && !(*stop)) { FileListInternEntry *entry; int nbr_entries = 0; @@ -3543,7 +3669,8 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib, if (filelist->asset_library_ref) { list_lib_options |= LIST_LIB_ASSETS_ONLY; } - nbr_entries = filelist_readjob_list_lib(subdir, &entries, list_lib_options); + nbr_entries = filelist_readjob_list_lib( + subdir, &entries, list_lib_options, &indexer_runtime); if (nbr_entries > 0) { is_lib = true; } @@ -3585,6 +3712,15 @@ static void filelist_readjob_recursive_dir_add_items(const bool do_lib, MEM_freeN(subdir); } + /* Finalize and free indexer. */ + if (indexer_runtime.callbacks->filelist_finished && BLI_stack_is_empty(todo_dirs)) { + indexer_runtime.callbacks->filelist_finished(indexer_runtime.user_data); + } + if (indexer_runtime.callbacks->free_user_data && indexer_runtime.user_data) { + indexer_runtime.callbacks->free_user_data(indexer_runtime.user_data); + indexer_runtime.user_data = NULL; + } + /* If we were interrupted by stop, stack may not be empty and we need to free * pending dir paths. */ while (!BLI_stack_is_empty(todo_dirs)) { diff --git a/source/blender/editors/space_file/filelist.h b/source/blender/editors/space_file/filelist.h index 0048a349dca..e7cda106224 100644 --- a/source/blender/editors/space_file/filelist.h +++ b/source/blender/editors/space_file/filelist.h @@ -29,6 +29,7 @@ extern "C" { struct AssetLibraryReference; struct BlendHandle; +struct FileIndexerType; struct FileList; struct FileSelection; struct bUUID; @@ -72,6 +73,7 @@ void filelist_setfilter_options(struct FileList *filelist, const bool filter_assets_only, const char *filter_glob, const char *filter_search); +void filelist_setindexer(struct FileList *filelist, const struct FileIndexerType *indexer); void filelist_set_asset_catalog_filter_options( struct FileList *filelist, eFileSel_Params_AssetCatalogVisibility catalog_visibility, diff --git a/source/blender/editors/space_file/space_file.c b/source/blender/editors/space_file/space_file.c index ef503708335..f2a01c9db6b 100644 --- a/source/blender/editors/space_file/space_file.c +++ b/source/blender/editors/space_file/space_file.c @@ -27,6 +27,7 @@ #include "MEM_guardedalloc.h" #include "BLI_blenlib.h" +#include "BLI_linklist.h" #include "BLI_utildefines.h" #include "BKE_appdir.h" @@ -43,6 +44,7 @@ #include "WM_message.h" #include "WM_types.h" +#include "ED_asset_indexer.h" #include "ED_asset.h" #include "ED_fileselect.h" #include "ED_screen.h" @@ -353,6 +355,10 @@ static void file_refresh(const bContext *C, ScrArea *area) sfile->files, asset_params->asset_catalog_visibility, &asset_params->catalog_id); } + if (ED_fileselect_is_asset_browser(sfile)) { + filelist_setindexer(sfile->files, &file_indexer_asset); + } + /* Update the active indices of bookmarks & co. */ sfile->systemnr = fsmenu_get_active_indices(fsmenu, FS_CATEGORY_SYSTEM, params->dir); sfile->system_bookmarknr = fsmenu_get_active_indices( |