Asset Catalogs: loading a catalog definition file

Initial, limited implementation of loading a single asset catalog
definition file. These files are structured as follows:

  CATALOG_ID virtual/path/of/catalog
  SUBCATALOG_ID virtual/path/of/catalog/child
  SOMETHING_ELSE some/unrelated/hierarchy

These virtual paths will be used to show the catalog in a tree
structure; the tree structure itself is not part of this commit. Each
asset will have one catalog ID that determines where in that tree the
asset shows up.

Currently only a single catalog definition file can be read; merging
data from multiple such files, and writing them out again after changes
are made, is for future commits.

This commit only contains the code to load a single file, and unittests
to check that this actually works. No UI, no user-facing functionality
yet.

4 files changed, 310 insertions, 0 deletions

diff --git a/source/blender/blenkernel/BKE_asset_catalog.hh b/source/blender/blenkernel/BKE_asset_catalog.hh
new file mode 100644
index 00000000000..82449bcdbcc
--- /dev/null
+++ b/source/blender/blenkernel/BKE_asset_catalog.hh
@@ -0,0 +1,97 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bke
+ */
+
+#pragma once
+
+#ifndef __cplusplus
+#  error This is a C++ header. The C interface is yet to be implemented/designed.
+#endif
+
+#include "BLI_map.hh"
+#include "BLI_set.hh"
+#include "BLI_string_ref.hh"
+#include "BLI_vector_set.hh"
+
+#include <filesystem>
+#include <memory>
+#include <string>
+
+namespace blender::bke {
+
+using CatalogID = std::string;
+using CatalogPath = std::string;
+using CatalogPathReference = StringRef;
+using CatalogPathComponent = std::string;
+using CatalogFilePath = std::filesystem::path;
+
+class AssetCatalog;
+class AssetCatalogDefinitionFile;
+class AssetCatalogTreeNode;
+
+/* Manages the asset catalogs of a single asset library (i.e. of catalogs defined in a single
+ * directory hierarchy). */
+class AssetCatalogService {
+  const char path_separator = '/';
+
+  /* TODO(@sybren): determine which properties should be private / get accessors. */
+
+  // These pointers are owned by this AssetCatalogService.
+  Map<CatalogID, std::unique_ptr<AssetCatalog>> catalogs;
+  std::unique_ptr<AssetCatalogDefinitionFile> catalog_definition_file;
+
+ public:
+  AssetCatalogService();
+
+  // Return nullptr if not found.
+  AssetCatalog *find_catalog(const CatalogID &catalog_id);
+
+  void load_from_disk(const CatalogFilePath &asset_library_root);
+
+ protected:
+  void load_directory_recursive(const CatalogFilePath &directory_path);
+  void load_single_file(const CatalogFilePath &catalog_definition_file_path);
+
+  std::unique_ptr<AssetCatalogDefinitionFile> parse_catalog_file(
+      const CatalogFilePath &catalog_definition_file_path);
+
+  std::unique_ptr<AssetCatalog> parse_catalog_line(
+      StringRef line, const AssetCatalogDefinitionFile *catalog_definition_file);
+};
+
+class AssetCatalogDefinitionFile {
+  /* TODO(@sybren): determine which properties should be private / get accessors. */
+ public:
+  CatalogFilePath file_path;
+  Map<CatalogID, AssetCatalog *> catalogs;
+
+  AssetCatalogDefinitionFile();
+};
+
+class AssetCatalog {
+  /* TODO(@sybren): determine which properties should be private / get accessors. */
+ public:
+  AssetCatalog();
+  AssetCatalog(const CatalogID &catalog_id, const CatalogPath &name);
+
+  CatalogID catalog_id;
+  CatalogPath path;
+};
+
+}  // namespace blender::bke
diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt
index 78bfe8c9afb..7c02ffbc105 100644
--- a/source/blender/blenkernel/CMakeLists.txt
+++ b/source/blender/blenkernel/CMakeLists.txt
@@ -82,6 +82,7 @@ set(SRC
   intern/armature_deform.c
   intern/armature_pose.cc
   intern/armature_update.c
+  intern/asset_catalog.cc
   intern/asset.cc
   intern/attribute.c
   intern/attribute_access.cc
@@ -298,6 +299,7 @@ set(SRC
   BKE_appdir.h
   BKE_armature.h
   BKE_armature.hh
+  BKE_asset_catalog.hh
   BKE_asset.h
   BKE_attribute.h
   BKE_attribute_access.hh
@@ -771,6 +773,7 @@ if(WITH_GTESTS)
   set(TEST_SRC
     intern/action_test.cc
     intern/armature_test.cc
+    intern/asset_catalog_test.cc
     intern/cryptomatte_test.cc
     intern/fcurve_test.cc
     intern/lattice_deform_test.cc
diff --git a/source/blender/blenkernel/intern/asset_catalog.cc b/source/blender/blenkernel/intern/asset_catalog.cc
new file mode 100644
index 00000000000..0139b7a4abf
--- /dev/null
+++ b/source/blender/blenkernel/intern/asset_catalog.cc
@@ -0,0 +1,148 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bke
+ */
+
+#include "BKE_asset_catalog.hh"
+
+#include "BLI_string_ref.hh"
+
+#include <filesystem>
+#include <fstream>
+
+namespace fs = std::filesystem;
+
+namespace blender::bke {
+
+AssetCatalogService::AssetCatalogService()
+{
+}
+
+AssetCatalog *AssetCatalogService::find_catalog(const CatalogID &catalog_id)
+{
+  std::unique_ptr<AssetCatalog> *catalog_uptr_ptr = this->catalogs.lookup_ptr(catalog_id);
+  if (catalog_uptr_ptr == nullptr) {
+    return nullptr;
+  }
+  return catalog_uptr_ptr->get();
+}
+
+void AssetCatalogService::load_from_disk(const CatalogFilePath &asset_library_root)
+{
+  fs::file_status status = fs::status(asset_library_root);
+  switch (status.type()) {
+    case fs::file_type::regular:
+      load_single_file(asset_library_root);
+      break;
+    case fs::file_type::directory:
+      load_directory_recursive(asset_library_root);
+      break;
+    default:
+      // TODO(@sybren): throw an appropriate exception.
+      return;
+  }
+}
+
+void AssetCatalogService::load_directory_recursive(const CatalogFilePath & /*directory_path*/)
+{
+  // TODO(@sybren): implement
+}
+
+void AssetCatalogService::load_single_file(const CatalogFilePath &catalog_definition_file_path)
+{
+  std::unique_ptr<AssetCatalogDefinitionFile> cdf = parse_catalog_file(
+      catalog_definition_file_path);
+
+  BLI_assert_msg(!this->catalog_definition_file,
+                 "Only loading of a single catalog definition file is supported.");
+  this->catalog_definition_file = std::move(cdf);
+}
+
+std::unique_ptr<AssetCatalogDefinitionFile> AssetCatalogService::parse_catalog_file(
+    const CatalogFilePath &catalog_definition_file_path)
+{
+  auto cdf = std::make_unique<AssetCatalogDefinitionFile>();
+  cdf->file_path = catalog_definition_file_path;
+
+  std::fstream infile(catalog_definition_file_path);
+  std::string line;
+  while (std::getline(infile, line)) {
+    const StringRef trimmed_line = StringRef(line).trim().trim(path_separator);
+    if (trimmed_line.is_empty() || trimmed_line[0] == '#') {
+      continue;
+    }
+
+    std::unique_ptr<AssetCatalog> catalog = this->parse_catalog_line(trimmed_line, cdf.get());
+
+    const bool is_catalog_id_reused_in_file = cdf->catalogs.contains(catalog->catalog_id);
+    /* The AssetDefinitionFile should include this catalog when writing it back to disk, even if it
+     * was a duplicate. */
+    cdf->catalogs.add_new(catalog->catalog_id, catalog.get());
+
+    if (is_catalog_id_reused_in_file) {
+      std::cerr << catalog_definition_file_path << ": multiple definitions of catalog "
+                << catalog->catalog_id << ", using first occurrence." << std::endl;
+      /* Don't store 'catalog'; unique_ptr will free its memory. */
+      continue;
+    }
+
+    if (this->catalogs.contains(catalog->catalog_id)) {
+      // TODO(@sybren): apparently another CDF was already loaded. This is not supported yet.
+      std::cerr << catalog_definition_file_path << ": multiple definitions of catalog "
+                << catalog->catalog_id << " in multiple files, ignoring this one." << std::endl;
+      /* Don't store 'catalog'; unique_ptr will free its memory. */
+      continue;
+    }
+
+    /* The AssetCatalog pointer is owned by the AssetCatalogService. */
+    this->catalogs.add_new(catalog->catalog_id, std::move(catalog));
+  }
+
+  return cdf;
+}
+
+std::unique_ptr<AssetCatalog> AssetCatalogService::parse_catalog_line(
+    const StringRef line, const AssetCatalogDefinitionFile *catalog_definition_file)
+{
+  const int64_t first_space = line.find_first_of(' ');
+  if (first_space == StringRef::not_found) {
+    std::cerr << "Invalid line in " << catalog_definition_file->file_path << ": " << line
+              << std::endl;
+    return std::unique_ptr<AssetCatalog>(nullptr);
+  }
+
+  const StringRef catalog_id = line.substr(0, first_space);
+  const StringRef catalog_path = line.substr(first_space + 1).trim().trim(path_separator);
+
+  return std::make_unique<AssetCatalog>(catalog_id, catalog_path);
+}
+
+AssetCatalogDefinitionFile::AssetCatalogDefinitionFile()
+{
+}
+
+AssetCatalog::AssetCatalog()
+{
+}
+
+AssetCatalog::AssetCatalog(const CatalogID &catalog_id, const CatalogPath &path)
+    : catalog_id(catalog_id), path(path)
+{
+}
+
+}  // namespace blender::bke
diff --git a/source/blender/blenkernel/intern/asset_catalog_test.cc b/source/blender/blenkernel/intern/asset_catalog_test.cc
new file mode 100644
index 00000000000..d6c00670bd6
--- /dev/null
+++ b/source/blender/blenkernel/intern/asset_catalog_test.cc
@@ -0,0 +1,62 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2020 Blender Foundation
+ * All rights reserved.
+ */
+
+#include "BKE_asset_catalog.hh"
+
+#include "testing/testing.h"
+
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+namespace blender::bke::tests {
+
+TEST(AssetCatalogTest, load_single_file)
+{
+  const fs::path test_files_dir = blender::tests::flags_test_asset_dir();
+  if (test_files_dir.empty()) {
+    FAIL();
+  }
+
+  AssetCatalogService service;
+  service.load_from_disk(test_files_dir / "asset_library/single_catalog_definition_file.cats.txt");
+
+  // Test getting a non-existant catalog ID.
+  EXPECT_EQ(nullptr, service.find_catalog("NONEXISTANT"));
+
+  // Test getting a 7-bit ASCII catalog ID.
+  AssetCatalog *poses_elly = service.find_catalog("POSES_ELLY");
+  ASSERT_NE(nullptr, poses_elly);
+  EXPECT_EQ("POSES_ELLY", poses_elly->catalog_id);
+  EXPECT_EQ("character/Elly/poselib", poses_elly->path);
+
+  // Test whitespace stripping.
+  AssetCatalog *poses_whitespace = service.find_catalog("POSES_ELLY_WHITESPACE");
+  ASSERT_NE(nullptr, poses_whitespace);
+  EXPECT_EQ("POSES_ELLY_WHITESPACE", poses_whitespace->catalog_id);
+  EXPECT_EQ("character/Elly/poselib/whitespace", poses_whitespace->path);
+
+  // Test getting a UTF-8 catalog ID.
+  AssetCatalog *poses_ruzena = service.find_catalog("POSES_RUŽENA");
+  ASSERT_NE(nullptr, poses_ruzena);
+  EXPECT_EQ("POSES_RUŽENA", poses_ruzena->catalog_id);
+  EXPECT_EQ("character/Ružena/poselib", poses_ruzena->path);
+}
+
+}  // namespace blender::bke::tests