Fix T93960: Asset Catalogs I/O fails with unicode file paths on Windows

On Windows, encode file paths as UTF-16 before trying to open the file
for reading/writing.

This introduces a new class `blender::fstream`, which wraps
`std::fstream` and provides this UTF-16 encoding. This class should also
be used in other areas, like the Alembic importer/exporter.

Manifest Task: T93960

Reviewed By: JacquesLucke

Differential Revision: https://developer.blender.org/D13633

4 files changed, 146 insertions, 0 deletions

diff --git a/source/blender/blenlib/BLI_fileops.hh b/source/blender/blenlib/BLI_fileops.hh
new file mode 100644
index 00000000000..c69b1983c59
--- /dev/null
+++ b/source/blender/blenlib/BLI_fileops.hh
@@ -0,0 +1,52 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bli
+ * \brief File and directory operations.
+ */
+
+#pragma once
+
+#ifndef __cplusplus
+#  error This is a C++ header
+#endif
+
+#include "BLI_fileops.h"
+#include "BLI_string_ref.hh"
+
+#include <fstream>
+#include <string>
+
+namespace blender {
+
+/**
+ * std::fstream subclass that handles UTF-16 encoding on Windows.
+ *
+ * For documentation, see https://en.cppreference.com/w/cpp/io/basic_fstream
+ */
+class fstream : public std::fstream {
+ public:
+  fstream() = default;
+  explicit fstream(const char *filepath,
+                   std::ios_base::openmode mode = ios_base::in | ios_base::out);
+  explicit fstream(const std::string &filepath,
+                   std::ios_base::openmode mode = ios_base::in | ios_base::out);
+
+  void open(StringRefNull filepath, ios_base::openmode mode = ios_base::in | ios_base::out);
+};
+
+}  // namespace blender
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index 29493c799b3..516d9d2fe84 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -76,6 +76,7 @@ set(SRC
   intern/endian_switch.c
   intern/expr_pylike_eval.c
   intern/fileops.c
+  intern/fileops.cc
   intern/filereader_file.c
   intern/filereader_gzip.c
   intern/filereader_memory.c
@@ -204,6 +205,7 @@ set(SRC
   BLI_enumerable_thread_specific.hh
   BLI_expr_pylike_eval.h
   BLI_fileops.h
+  BLI_fileops.hh
   BLI_fileops_types.h
   BLI_filereader.h
   BLI_float2.hh
@@ -422,6 +424,7 @@ if(WITH_GTESTS)
     tests/BLI_edgehash_test.cc
     tests/BLI_expr_pylike_eval_test.cc
     tests/BLI_function_ref_test.cc
+    tests/BLI_fileops_test.cc
     tests/BLI_ghash_test.cc
     tests/BLI_hash_mm2a_test.cc
     tests/BLI_heap_simple_test.cc
diff --git a/source/blender/blenlib/intern/fileops.cc b/source/blender/blenlib/intern/fileops.cc
new file mode 100644
index 00000000000..5ceedbd8cb5
--- /dev/null
+++ b/source/blender/blenlib/intern/fileops.cc
@@ -0,0 +1,51 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/** \file
+ * \ingroup bli
+ */
+
+#include "BLI_fileops.hh"
+
+#ifdef WIN32
+#  include "utfconv.h"
+#endif
+
+namespace blender {
+fstream::fstream(const char *filepath, std::ios_base::openmode mode)
+{
+  this->open(filepath, mode);
+}
+
+fstream::fstream(const std::string &filepath, std::ios_base::openmode mode)
+{
+  this->open(filepath, mode);
+}
+
+void fstream::open(StringRefNull filepath, ios_base::openmode mode)
+{
+#ifdef WIN32
+  const char *filepath_cstr = filepath.c_str();
+  UTF16_ENCODE(filepath_cstr);
+  std::wstring filepath_wstr(filepath_cstr_16);
+  std::fstream::open(filepath_wstr.c_str(), mode);
+  UTF16_UN_ENCODE(filepath_cstr);
+#else
+  std::fstream::open(filepath, mode);
+#endif
+}
+
+}  // namespace blender
diff --git a/source/blender/blenlib/tests/BLI_fileops_test.cc b/source/blender/blenlib/tests/BLI_fileops_test.cc
new file mode 100644
index 00000000000..e2a792647dc
--- /dev/null
+++ b/source/blender/blenlib/tests/BLI_fileops_test.cc
@@ -0,0 +1,40 @@
+/* Apache License, Version 2.0 */
+
+#include "BLI_fileops.hh"
+
+#include "testing/testing.h"
+
+namespace blender::tests {
+
+TEST(fileops, fstream_open_string_filename)
+{
+  const std::string test_files_dir = blender::tests::flags_test_asset_dir();
+  if (test_files_dir.empty()) {
+    FAIL();
+  }
+
+  const std::string filepath = test_files_dir + "/asset_library/новый/blender_assets.cats.txt";
+  fstream in(filepath, std::ios_base::in);
+  ASSERT_TRUE(in.is_open()) << "could not open " << filepath;
+  in.close(); /* This should not crash. */
+
+  /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */
+}
+
+TEST(fileops, fstream_open_charptr_filename)
+{
+  const std::string test_files_dir = blender::tests::flags_test_asset_dir();
+  if (test_files_dir.empty()) {
+    FAIL();
+  }
+
+  const std::string filepath_str = test_files_dir + "/asset_library/новый/blender_assets.cats.txt";
+  const char *filepath = filepath_str.c_str();
+  fstream in(filepath, std::ios_base::in);
+  ASSERT_TRUE(in.is_open()) << "could not open " << filepath;
+  in.close(); /* This should not crash. */
+
+  /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */
+}
+
+}  // namespace blender::tests