Fix T93960: Asset Catalogs I/O fails with unicode file paths on Windows

On Windows, encode file paths as UTF-16 before trying to open the file
for reading/writing.

This introduces a new class `blender::fstream`, which wraps
`std::fstream` and provides this UTF-16 encoding. This class should also
be used in other areas, like the Alembic importer/exporter.

Manifest Task: T93960

Reviewed By: JacquesLucke

Differential Revision: https://developer.blender.org/D13633

6 files changed, 173 insertions, 3 deletions

diff --git a/source/blender/blenkernel/intern/asset_catalog.cc b/source/blender/blenkernel/intern/asset_catalog.cc
index 79504edb50d..eee1f6287c3 100644
--- a/source/blender/blenkernel/intern/asset_catalog.cc
+++ b/source/blender/blenkernel/intern/asset_catalog.cc
@@ -24,7 +24,7 @@
 #include "BKE_asset_catalog.hh"
 #include "BKE_asset_library.h"
 
-#include "BLI_fileops.h"
+#include "BLI_fileops.hh"
 #include "BLI_path_util.h"
 
 /* For S_ISREG() and S_ISDIR() on Windows. */
@@ -830,7 +830,7 @@ void AssetCatalogDefinitionFile::parse_catalog_file(
     const CatalogFilePath &catalog_definition_file_path,
     AssetCatalogParsedFn catalog_loaded_callback)
 {
-  std::fstream infile(catalog_definition_file_path);
+  fstream infile(catalog_definition_file_path, std::ios::in);
 
   if (!infile.is_open()) {
     CLOG_ERROR(&LOG, "%s: unable to open file", catalog_definition_file_path.c_str());
@@ -966,7 +966,7 @@ bool AssetCatalogDefinitionFile::write_to_disk_unsafe(const CatalogFilePath &des
     return false;
   }
 
-  std::ofstream output(dest_file_path);
+  fstream output(dest_file_path, std::ios::out);
 
   /* TODO(@sybren): remember the line ending style that was originally read, then use that to write
    * the file again. */
diff --git a/source/blender/blenkernel/intern/asset_catalog_test.cc b/source/blender/blenkernel/intern/asset_catalog_test.cc
index 7c82b100119..3ff7831b19a 100644
--- a/source/blender/blenkernel/intern/asset_catalog_test.cc
+++ b/source/blender/blenkernel/intern/asset_catalog_test.cc
@@ -563,6 +563,30 @@ TEST_F(AssetCatalogTest, write_single_file)
   /* TODO(@sybren): test ordering of catalogs in the file. */
 }
 
+TEST_F(AssetCatalogTest, read_write_unicode_filepath)
+{
+  TestableAssetCatalogService service(asset_library_root_);
+  const CatalogFilePath load_from_path = asset_library_root_ + "/новый/" +
+                                         AssetCatalogService::DEFAULT_CATALOG_FILENAME;
+  service.load_from_disk(load_from_path);
+
+  const CatalogFilePath save_to_path = use_temp_path() + "новый.cats.txt";
+  AssetCatalogDefinitionFile *cdf = service.get_catalog_definition_file();
+  ASSERT_NE(nullptr, cdf) << "unable to load " << load_from_path;
+  EXPECT_TRUE(cdf->write_to_disk(save_to_path));
+
+  AssetCatalogService loaded_service(save_to_path);
+  loaded_service.load_from_disk();
+
+  /* Test that the file was loaded correctly. */
+  const bUUID materials_uuid("a2151dff-dead-4f29-b6bc-b2c7d6cccdb4");
+  const AssetCatalog *cat = loaded_service.find_catalog(materials_uuid);
+  ASSERT_NE(nullptr, cat);
+  EXPECT_EQ(materials_uuid, cat->catalog_id);
+  EXPECT_EQ(AssetCatalogPath("Материалы"), cat->path);
+  EXPECT_EQ("Russian Materials", cat->simple_name);
+}
+
 TEST_F(AssetCatalogTest, no_writing_empty_files)
 {
   const CatalogFilePath temp_lib_root = create_temp_path();
diff --git a/source/blender/blenlib/BLI_fileops.hh b/source/blender/blenlib/BLI_fileops.hh
new file mode 100644
index 00000000000..c69b1983c59
--- /dev/null
+++ b/source/blender/blenlib/BLI_fileops.hh
@@ -0,0 +1,52 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bli
+ * \brief File and directory operations.
+ */
+
+#pragma once
+
+#ifndef __cplusplus
+#  error This is a C++ header
+#endif
+
+#include "BLI_fileops.h"
+#include "BLI_string_ref.hh"
+
+#include <fstream>
+#include <string>
+
+namespace blender {
+
+/**
+ * std::fstream subclass that handles UTF-16 encoding on Windows.
+ *
+ * For documentation, see https://en.cppreference.com/w/cpp/io/basic_fstream
+ */
+class fstream : public std::fstream {
+ public:
+  fstream() = default;
+  explicit fstream(const char *filepath,
+                   std::ios_base::openmode mode = ios_base::in | ios_base::out);
+  explicit fstream(const std::string &filepath,
+                   std::ios_base::openmode mode = ios_base::in | ios_base::out);
+
+  void open(StringRefNull filepath, ios_base::openmode mode = ios_base::in | ios_base::out);
+};
+
+}  // namespace blender
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index 29493c799b3..516d9d2fe84 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -76,6 +76,7 @@ set(SRC
   intern/endian_switch.c
   intern/expr_pylike_eval.c
   intern/fileops.c
+  intern/fileops.cc
   intern/filereader_file.c
   intern/filereader_gzip.c
   intern/filereader_memory.c
@@ -204,6 +205,7 @@ set(SRC
   BLI_enumerable_thread_specific.hh
   BLI_expr_pylike_eval.h
   BLI_fileops.h
+  BLI_fileops.hh
   BLI_fileops_types.h
   BLI_filereader.h
   BLI_float2.hh
@@ -422,6 +424,7 @@ if(WITH_GTESTS)
     tests/BLI_edgehash_test.cc
     tests/BLI_expr_pylike_eval_test.cc
     tests/BLI_function_ref_test.cc
+    tests/BLI_fileops_test.cc
     tests/BLI_ghash_test.cc
     tests/BLI_hash_mm2a_test.cc
     tests/BLI_heap_simple_test.cc
diff --git a/source/blender/blenlib/intern/fileops.cc b/source/blender/blenlib/intern/fileops.cc
new file mode 100644
index 00000000000..5ceedbd8cb5
--- /dev/null
+++ b/source/blender/blenlib/intern/fileops.cc
@@ -0,0 +1,51 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include "BLI_fileops.hh"
+
+#ifdef WIN32
+#  include "utfconv.h"
+#endif
+
+namespace blender {
+fstream::fstream(const char *filepath, std::ios_base::openmode mode)
+{
+  this->open(filepath, mode);
+}
+
+fstream::fstream(const std::string &filepath, std::ios_base::openmode mode)
+{
+  this->open(filepath, mode);
+}
+
+void fstream::open(StringRefNull filepath, ios_base::openmode mode)
+{
+#ifdef WIN32
+  const char *filepath_cstr = filepath.c_str();
+  UTF16_ENCODE(filepath_cstr);
+  std::wstring filepath_wstr(filepath_cstr_16);
+  std::fstream::open(filepath_wstr.c_str(), mode);
+  UTF16_UN_ENCODE(filepath_cstr);
+#else
+  std::fstream::open(filepath, mode);
+#endif
+}
+
+}  // namespace blender
diff --git a/source/blender/blenlib/tests/BLI_fileops_test.cc b/source/blender/blenlib/tests/BLI_fileops_test.cc
new file mode 100644
index 00000000000..e2a792647dc
--- /dev/null
+++ b/source/blender/blenlib/tests/BLI_fileops_test.cc
@@ -0,0 +1,40 @@
+/* Apache License, Version 2.0 */
+
+#include "BLI_fileops.hh"
+
+#include "testing/testing.h"
+
+namespace blender::tests {
+
+TEST(fileops, fstream_open_string_filename)
+{
+  const std::string test_files_dir = blender::tests::flags_test_asset_dir();
+  if (test_files_dir.empty()) {
+    FAIL();
+  }
+
+  const std::string filepath = test_files_dir + "/asset_library/новый/blender_assets.cats.txt";
+  fstream in(filepath, std::ios_base::in);
+  ASSERT_TRUE(in.is_open()) << "could not open " << filepath;
+  in.close(); /* This should not crash. */
+
+  /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */
+}
+
+TEST(fileops, fstream_open_charptr_filename)
+{
+  const std::string test_files_dir = blender::tests::flags_test_asset_dir();
+  if (test_files_dir.empty()) {
+    FAIL();
+  }
+
+  const std::string filepath_str = test_files_dir + "/asset_library/новый/blender_assets.cats.txt";
+  const char *filepath = filepath_str.c_str();
+  fstream in(filepath, std::ios_base::in);
+  ASSERT_TRUE(in.is_open()) << "could not open " << filepath;
+  in.close(); /* This should not crash. */
+
+  /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */
+}
+
+}  // namespace blender::tests