From 8cf19944557452ae7a9c1cb2365d6121f2dfdb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 21 Dec 2021 15:53:52 +0100 Subject: Fix T93960: Asset Catalogs I/O fails with unicode file paths on Windows On Windows, encode file paths as UTF-16 before trying to open the file for reading/writing. This introduces a new class `blender::fstream`, which wraps `std::fstream` and provides this UTF-16 encoding. This class should also be used in other areas, like the Alembic importer/exporter. Manifest Task: T93960 Reviewed By: JacquesLucke Differential Revision: https://developer.blender.org/D13633 --- source/blender/blenkernel/intern/asset_catalog.cc | 6 +-- .../blenkernel/intern/asset_catalog_test.cc | 24 ++++++++++ source/blender/blenlib/BLI_fileops.hh | 52 ++++++++++++++++++++++ source/blender/blenlib/CMakeLists.txt | 3 ++ source/blender/blenlib/intern/fileops.cc | 51 +++++++++++++++++++++ source/blender/blenlib/tests/BLI_fileops_test.cc | 40 +++++++++++++++++ 6 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 source/blender/blenlib/BLI_fileops.hh create mode 100644 source/blender/blenlib/intern/fileops.cc create mode 100644 source/blender/blenlib/tests/BLI_fileops_test.cc (limited to 'source/blender') diff --git a/source/blender/blenkernel/intern/asset_catalog.cc b/source/blender/blenkernel/intern/asset_catalog.cc index 79504edb50d..eee1f6287c3 100644 --- a/source/blender/blenkernel/intern/asset_catalog.cc +++ b/source/blender/blenkernel/intern/asset_catalog.cc @@ -24,7 +24,7 @@ #include "BKE_asset_catalog.hh" #include "BKE_asset_library.h" -#include "BLI_fileops.h" +#include "BLI_fileops.hh" #include "BLI_path_util.h" /* For S_ISREG() and S_ISDIR() on Windows. */ @@ -830,7 +830,7 @@ void AssetCatalogDefinitionFile::parse_catalog_file( const CatalogFilePath &catalog_definition_file_path, AssetCatalogParsedFn catalog_loaded_callback) { - std::fstream infile(catalog_definition_file_path); + fstream infile(catalog_definition_file_path, std::ios::in); if (!infile.is_open()) { CLOG_ERROR(&LOG, "%s: unable to open file", catalog_definition_file_path.c_str()); @@ -966,7 +966,7 @@ bool AssetCatalogDefinitionFile::write_to_disk_unsafe(const CatalogFilePath &des return false; } - std::ofstream output(dest_file_path); + fstream output(dest_file_path, std::ios::out); /* TODO(@sybren): remember the line ending style that was originally read, then use that to write * the file again. */ diff --git a/source/blender/blenkernel/intern/asset_catalog_test.cc b/source/blender/blenkernel/intern/asset_catalog_test.cc index 7c82b100119..3ff7831b19a 100644 --- a/source/blender/blenkernel/intern/asset_catalog_test.cc +++ b/source/blender/blenkernel/intern/asset_catalog_test.cc @@ -563,6 +563,30 @@ TEST_F(AssetCatalogTest, write_single_file) /* TODO(@sybren): test ordering of catalogs in the file. */ } +TEST_F(AssetCatalogTest, read_write_unicode_filepath) +{ + TestableAssetCatalogService service(asset_library_root_); + const CatalogFilePath load_from_path = asset_library_root_ + "/новый/" + + AssetCatalogService::DEFAULT_CATALOG_FILENAME; + service.load_from_disk(load_from_path); + + const CatalogFilePath save_to_path = use_temp_path() + "новый.cats.txt"; + AssetCatalogDefinitionFile *cdf = service.get_catalog_definition_file(); + ASSERT_NE(nullptr, cdf) << "unable to load " << load_from_path; + EXPECT_TRUE(cdf->write_to_disk(save_to_path)); + + AssetCatalogService loaded_service(save_to_path); + loaded_service.load_from_disk(); + + /* Test that the file was loaded correctly. */ + const bUUID materials_uuid("a2151dff-dead-4f29-b6bc-b2c7d6cccdb4"); + const AssetCatalog *cat = loaded_service.find_catalog(materials_uuid); + ASSERT_NE(nullptr, cat); + EXPECT_EQ(materials_uuid, cat->catalog_id); + EXPECT_EQ(AssetCatalogPath("Материалы"), cat->path); + EXPECT_EQ("Russian Materials", cat->simple_name); +} + TEST_F(AssetCatalogTest, no_writing_empty_files) { const CatalogFilePath temp_lib_root = create_temp_path(); diff --git a/source/blender/blenlib/BLI_fileops.hh b/source/blender/blenlib/BLI_fileops.hh new file mode 100644 index 00000000000..c69b1983c59 --- /dev/null +++ b/source/blender/blenlib/BLI_fileops.hh @@ -0,0 +1,52 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup bli + * \brief File and directory operations. + */ + +#pragma once + +#ifndef __cplusplus +# error This is a C++ header +#endif + +#include "BLI_fileops.h" +#include "BLI_string_ref.hh" + +#include +#include + +namespace blender { + +/** + * std::fstream subclass that handles UTF-16 encoding on Windows. + * + * For documentation, see https://en.cppreference.com/w/cpp/io/basic_fstream + */ +class fstream : public std::fstream { + public: + fstream() = default; + explicit fstream(const char *filepath, + std::ios_base::openmode mode = ios_base::in | ios_base::out); + explicit fstream(const std::string &filepath, + std::ios_base::openmode mode = ios_base::in | ios_base::out); + + void open(StringRefNull filepath, ios_base::openmode mode = ios_base::in | ios_base::out); +}; + +} // namespace blender diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt index 29493c799b3..516d9d2fe84 100644 --- a/source/blender/blenlib/CMakeLists.txt +++ b/source/blender/blenlib/CMakeLists.txt @@ -76,6 +76,7 @@ set(SRC intern/endian_switch.c intern/expr_pylike_eval.c intern/fileops.c + intern/fileops.cc intern/filereader_file.c intern/filereader_gzip.c intern/filereader_memory.c @@ -204,6 +205,7 @@ set(SRC BLI_enumerable_thread_specific.hh BLI_expr_pylike_eval.h BLI_fileops.h + BLI_fileops.hh BLI_fileops_types.h BLI_filereader.h BLI_float2.hh @@ -422,6 +424,7 @@ if(WITH_GTESTS) tests/BLI_edgehash_test.cc tests/BLI_expr_pylike_eval_test.cc tests/BLI_function_ref_test.cc + tests/BLI_fileops_test.cc tests/BLI_ghash_test.cc tests/BLI_hash_mm2a_test.cc tests/BLI_heap_simple_test.cc diff --git a/source/blender/blenlib/intern/fileops.cc b/source/blender/blenlib/intern/fileops.cc new file mode 100644 index 00000000000..5ceedbd8cb5 --- /dev/null +++ b/source/blender/blenlib/intern/fileops.cc @@ -0,0 +1,51 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** \file + * \ingroup bli + */ + +#include "BLI_fileops.hh" + +#ifdef WIN32 +# include "utfconv.h" +#endif + +namespace blender { +fstream::fstream(const char *filepath, std::ios_base::openmode mode) +{ + this->open(filepath, mode); +} + +fstream::fstream(const std::string &filepath, std::ios_base::openmode mode) +{ + this->open(filepath, mode); +} + +void fstream::open(StringRefNull filepath, ios_base::openmode mode) +{ +#ifdef WIN32 + const char *filepath_cstr = filepath.c_str(); + UTF16_ENCODE(filepath_cstr); + std::wstring filepath_wstr(filepath_cstr_16); + std::fstream::open(filepath_wstr.c_str(), mode); + UTF16_UN_ENCODE(filepath_cstr); +#else + std::fstream::open(filepath, mode); +#endif +} + +} // namespace blender diff --git a/source/blender/blenlib/tests/BLI_fileops_test.cc b/source/blender/blenlib/tests/BLI_fileops_test.cc new file mode 100644 index 00000000000..e2a792647dc --- /dev/null +++ b/source/blender/blenlib/tests/BLI_fileops_test.cc @@ -0,0 +1,40 @@ +/* Apache License, Version 2.0 */ + +#include "BLI_fileops.hh" + +#include "testing/testing.h" + +namespace blender::tests { + +TEST(fileops, fstream_open_string_filename) +{ + const std::string test_files_dir = blender::tests::flags_test_asset_dir(); + if (test_files_dir.empty()) { + FAIL(); + } + + const std::string filepath = test_files_dir + "/asset_library/новый/blender_assets.cats.txt"; + fstream in(filepath, std::ios_base::in); + ASSERT_TRUE(in.is_open()) << "could not open " << filepath; + in.close(); /* This should not crash. */ + + /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */ +} + +TEST(fileops, fstream_open_charptr_filename) +{ + const std::string test_files_dir = blender::tests::flags_test_asset_dir(); + if (test_files_dir.empty()) { + FAIL(); + } + + const std::string filepath_str = test_files_dir + "/asset_library/новый/blender_assets.cats.txt"; + const char *filepath = filepath_str.c_str(); + fstream in(filepath, std::ios_base::in); + ASSERT_TRUE(in.is_open()) << "could not open " << filepath; + in.close(); /* This should not crash. */ + + /* Reading the file not tested here. That's deferred to `std::fstream` anyway. */ +} + +} // namespace blender::tests -- cgit v1.2.3