diff options
author | Niall Douglas (s [underscore] sourceforge {at} nedprod [dot] com) <spamtrap@nedprod.com> | 2021-12-01 14:58:34 +0300 |
---|---|---|
committer | Niall Douglas (s [underscore] sourceforge {at} nedprod [dot] com) <spamtrap@nedprod.com> | 2021-12-01 14:58:34 +0300 |
commit | 6b41d028c9d84fed31dadc1b291cf8f12db1c405 (patch) | |
tree | b101516f1214e395da253b672386bd5a4bddc960 | |
parent | 73dd97832f1e115531d76c4f82d11391e4450650 (diff) | |
parent | 713b9dc878eeaf457e990386414466b8c4e334d5 (diff) |
Merge branch 'develop' into dynamic_thread_pool_groupdynamic_thread_pool_group
42 files changed, 1276 insertions, 618 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 4293966f..ddfe3e7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,18 @@ # http://www.boost.org/LICENSE_1_0.txt) cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +set(LLFIO_DEPENDENCY_QUICKCPPLIB_GIT_TAG "master" CACHE STRING "Which git tag to use for the QuickCppLib dependency") +set(LLFIO_DEPENDENCY_OUTCOME_GIT_TAG "master" CACHE STRING "Which git tag to use for the Outcome dependency") +if(NOT LLFIO_DEPENDENCY_QUICKCPPLIB_GIT_TAG STREQUAL "master") + # Don't reuse the bootstrapped git clone as the dependency itself + if(CMAKE_BINARY_DIR) + set(CTEST_QUICKCPPLIB_CLONE_DIR "${CMAKE_BINARY_DIR}/quickcpplib-bootstrap") + else() + set(CTEST_QUICKCPPLIB_CLONE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/quickcpplib-bootstrap") + endif() +endif() + include(cmake/QuickCppLibBootstrap.cmake) include(QuickCppLibRequireOutOfSourceBuild) include(QuickCppLibUtils) @@ -30,6 +42,8 @@ include(QuickCppLibPolicies) option(LLFIO_USE_EXPERIMENTAL_SG14_STATUS_CODE "Whether to use SG14 status_code for failure handling" OFF) option(LLFIO_ENABLE_DEPENDENCY_SMOKE_TEST "Whether to build executables which are smoke tests that LLFIO is fully working. Used by various package managers such as vcpkg." OFF) option(LLFIO_ASSUME_CROSS_COMPILING "Whether to assume we are cross compiling. Normally automatically detected, but if automatic detection doesn't work, a working <filesystem> will not be found during cmake configure." OFF) +option(LLFIO_FORCE_CONCEPTS_OFF "Whether to not auto detect and enable concepts for the LLFIO cmake targets" OFF) +option(LLFIO_FORCE_COROUTINES_OFF "Whether to not auto detect and enable coroutines for the LLFIO cmake targets" OFF) option(UNIT_TESTS_BUILD_ALL "Whether to run all of the unit test suite." OFF) set(UNIT_TESTS_CXX_VERSION "latest" CACHE STRING "The version of C++ to use in the header-only unit tests") if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" OR APPLE) @@ -55,12 +69,13 @@ endif() # Find my library dependencies find_quickcpplib_library(quickcpplib GIT_REPOSITORY "https://github.com/ned14/quickcpplib.git" + GIT_TAG "${LLFIO_DEPENDENCY_QUICKCPPLIB_GIT_TAG}" REQUIRED IS_HEADER_ONLY ) find_quickcpplib_library(outcome GIT_REPOSITORY "https://github.com/ned14/outcome.git" - GIT_TAG "master" + GIT_TAG "${LLFIO_DEPENDENCY_OUTCOME_GIT_TAG}" REQUIRED IS_HEADER_ONLY ) @@ -84,12 +99,16 @@ include(QuickCppLibMakeLibrary) include(QuickCppLibMakeHeaderOnlyLibrary) # If we have concepts, enable those for both myself and all inclusions -apply_cxx_concepts_to(INTERFACE llfio_hl) -apply_cxx_concepts_to(PUBLIC llfio_sl llfio_dl) +if(NOT LLFIO_FORCE_CONCEPTS_OFF) + apply_cxx_concepts_to(INTERFACE llfio_hl) + apply_cxx_concepts_to(PUBLIC llfio_sl llfio_dl) +endif() # If we have coroutines, enable those for both myself and all inclusions -apply_cxx_coroutines_to(INTERFACE llfio_hl) -apply_cxx_coroutines_to(PUBLIC llfio_sl llfio_dl) +if(NOT LLFIO_FORCE_COROUTINES_OFF) + apply_cxx_coroutines_to(INTERFACE llfio_hl) + apply_cxx_coroutines_to(PUBLIC llfio_sl llfio_dl) +endif() # Make preprocessed edition of this library target if(NOT PROJECT_IS_DEPENDENCY) diff --git a/cmake/QuickCppLibBootstrap.cmake b/cmake/QuickCppLibBootstrap.cmake index 359c0eb7..292a5a98 100644 --- a/cmake/QuickCppLibBootstrap.cmake +++ b/cmake/QuickCppLibBootstrap.cmake @@ -53,6 +53,9 @@ if(NOT quickcpplib_done) set(CTEST_QUICKCPPLIB_SCRIPTS "${CMAKE_SOURCE_DIR}/../quickcpplib/scripts") # Copy latest version of myself into end user file(COPY "${CTEST_QUICKCPPLIB_SCRIPTS}/../cmake/QuickCppLibBootstrap.cmake" DESTINATION "${CMAKE_SOURCE_DIR}/cmake/") + elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../.quickcpplib_use_siblings" AND NOT QUICKCPPLIB_DISABLE_SIBLINGS) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../quickcpplib/cmakelib") + set(CTEST_QUICKCPPLIB_SCRIPTS "${CMAKE_CURRENT_SOURCE_DIR}/../quickcpplib/scripts") elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/quickcpplib/repo/cmakelib") set(CTEST_QUICKCPPLIB_CLONE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/quickcpplib") elseif(CMAKE_BINARY_DIR) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index c8eabb3a..9201bd4f 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -22,6 +22,7 @@ set(llfio_TESTS "test/tests/map_handle_create_close/kernel_map_handle.cpp.hpp" "test/tests/map_handle_create_close/runner.cpp" "test/tests/mapped.cpp" + "test/tests/mapped_file_handle.cpp" "test/tests/path_discovery.cpp" "test/tests/path_view.cpp" "test/tests/pipe_handle.cpp" diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp index 9d544a02..9c627e88 100644 --- a/include/llfio/revision.hpp +++ b/include/llfio/revision.hpp @@ -1,4 +1,4 @@ // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time -#define LLFIO_PREVIOUS_COMMIT_REF 46f0760e7ed2b80c46acd91bacc130049620bee6 -#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-21 13:31:38 +00:00" -#define LLFIO_PREVIOUS_COMMIT_UNIQUE 46f0760e +#define LLFIO_PREVIOUS_COMMIT_REF d3c51113c70ee9a44b6d925078058c2a7af9b3bc +#define LLFIO_PREVIOUS_COMMIT_DATE "2021-12-01 11:46:19 +00:00" +#define LLFIO_PREVIOUS_COMMIT_UNIQUE d3c51113 diff --git a/include/llfio/v2.0/detail/impl/map_handle.ipp b/include/llfio/v2.0/detail/impl/map_handle.ipp index e5e9fa94..16c34adf 100644 --- a/include/llfio/v2.0/detail/impl/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/map_handle.ipp @@ -53,13 +53,17 @@ namespace detail }; struct map_handle_cache_base_t { - std::mutex lock; + mutable std::mutex lock; size_t trie_count{0}; map_handle_cache_item_t *trie_children[8 * sizeof(size_t)]; - bool trie_nobbledir{0}; + bool trie_nobbledir{0}, disabled{false}; size_t bytes_in_cache{0}, hits{0}, misses{0}; }; - static const size_t page_size_shift = [] { return QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::detail::bitscanr(utils::page_size()); }(); + inline size_t page_size_shift() + { + static size_t v = [] { return QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::detail::bitscanr(utils::page_size()); }(); + return v; + } class map_handle_cache_t : protected QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie<map_handle_cache_base_t, map_handle_cache_item_t> { using _base = QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie<map_handle_cache_base_t, map_handle_cache_item_t>; @@ -70,13 +74,20 @@ namespace detail std::atomic<unsigned> do_not_store_failed_count{0}; #endif - ~map_handle_cache_t() { trim_cache(std::chrono::steady_clock::now(), (size_t)-1); } + ~map_handle_cache_t() { trim_cache(std::chrono::steady_clock::now(), (size_t) -1); } + + bool is_disabled() const noexcept + { + _lock_guard g(lock); + return _base::disabled; + } using _base::size; void *get(size_t bytes, size_t page_size) { - const auto _bytes = bytes >> page_size_shift; + const auto _bytes = bytes >> page_size_shift(); _lock_guard g(lock); + // TODO: Consider finding slightly bigger, and returning a length shorter than reservation? auto it = _base::find(_bytes); for(; it != _base::end() && page_size != it->page_size && _bytes == it->trie_key; ++it) { @@ -90,19 +101,26 @@ namespace detail auto *p = *it; _base::erase(it); _base::bytes_in_cache -= bytes; - assert(bytes == p->trie_key << page_size_shift); + assert(bytes == p->trie_key << page_size_shift()); + // std::cout << "map_handle::get(" << p->addr << ", " << bytes << "). Index item was " << p << std::endl; g.unlock(); void *ret = p->addr; delete p; return ret; } - void add(size_t bytes, size_t page_size, void *addr) + bool add(size_t bytes, size_t page_size, void *addr) { - const auto _bytes = bytes >> page_size_shift; + const auto _bytes = bytes >> page_size_shift(); + if(_bytes == 0) + { + return false; + } auto *p = new map_handle_cache_item_t(_bytes, page_size, addr); _lock_guard g(lock); + // std::cout << "map_handle::add(" << addr << ", " << bytes << "). Index item was " << p << ". Trie key is " << _bytes << std::endl; _base::insert(p); _base::bytes_in_cache += bytes; + return true; } map_handle::cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than, size_t max_items) { @@ -118,15 +136,18 @@ namespace detail { auto *p = *it; _base::erase(it--); - const auto _bytes = p->trie_key << page_size_shift; + const auto _bytes = p->trie_key << page_size_shift(); + // std::cout << "map_handle::trim_cache(" << p->addr << ", " << _bytes << "). Index item was " << p << ". Trie key is " << p->trie_key << std::endl; #ifdef _WIN32 if(!win32_release_nonfile_allocations((byte *) p->addr, _bytes, MEM_RELEASE)) #else if(-1 == ::munmap(p->addr, _bytes)) #endif { + // fprintf(stderr, "munmap failed with %s. addr was %p bytes was %zu. page_size_shift was %zu\n", strerror(errno), p->addr, _bytes, + // page_size_shift); LLFIO_LOG_FATAL(nullptr, - "map_handle cache failed to trim a map! If on Linux, you may have exceeded the " + "FATAL: map_handle cache failed to trim a map! If on Linux, you may have exceeded the " "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); abort(); @@ -149,11 +170,36 @@ namespace detail ret.misses = _base::misses; return ret; } + bool set_cache_disabled(bool v) + { + _lock_guard g(lock); + bool ret = _base::disabled; + _base::disabled = v; + return ret; + } }; - extern inline QUICKCPPLIB_SYMBOL_EXPORT map_handle_cache_t &map_handle_cache() + extern inline QUICKCPPLIB_SYMBOL_VISIBLE map_handle_cache_t *map_handle_cache() { - static map_handle_cache_t v; - return v; + static struct _ + { + map_handle_cache_t *v; + _() + : v(new map_handle_cache_t) + { + } + ~_() + { + if(v != nullptr) + { + // std::cout << "map_handle_cache static deinit begin" << std::endl; + auto *r = v; + *(volatile map_handle_cache_t **) &v = nullptr; + delete r; + // std::cout << "map_handle_cache static deinit complete" << std::endl; + } + } + } v; + return v.v; } } // namespace detail @@ -163,12 +209,17 @@ result<map_handle> map_handle::_recycled_map(size_type bytes, section_handle::fl { return errc::argument_out_of_domain; } + auto *c = detail::map_handle_cache(); + if(c == nullptr || c->is_disabled() || bytes >= (1ULL << 30U) /*1Gb*/) + { + return _new_map(bytes, false, _flag); + } result<map_handle> ret(map_handle(nullptr, _flag)); native_handle_type &nativeh = ret.value()._v; OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag)); bytes = utils::round_up_to_page_size(bytes, pagesize); LLFIO_LOG_FUNCTION_CALL(&ret); - void *addr = detail::map_handle_cache().get(bytes, pagesize); + void *addr = c->get(bytes, pagesize); if(addr == nullptr) { return _new_map(bytes, false, _flag); @@ -224,36 +275,41 @@ bool map_handle::_recycle_map() noexcept try { LLFIO_LOG_FUNCTION_CALL(this); - auto &c = detail::map_handle_cache(); + auto *c = detail::map_handle_cache(); + if(c == nullptr || c->is_disabled() || _reservation >= (1ULL << 30U) /*1Gb*/) + { + return false; + } #ifdef _WIN32 if(!win32_release_nonfile_allocations(_addr, _length, MEM_DECOMMIT)) { return false; } #else +#ifdef __linux__ if(memory_accounting() == memory_accounting_kind::commit_charge) +#endif { if(!do_mmap(_v, _addr, MAP_FIXED, nullptr, _pagesize, _length, 0, section_handle::flag::none | section_handle::flag::nocommit)) { return false; } } +#ifdef __linux__ else { -#ifdef __linux__ - if(c.do_not_store_failed_count.load(std::memory_order_relaxed) < 10) + if(c->do_not_store_failed_count.load(std::memory_order_relaxed) < 10) { auto r = do_not_store({_addr, _length}); - if(!r) + if(!r || r.assume_value().size() == 0) { - c.do_not_store_failed_count.fetch_add(1, std::memory_order_relaxed); + c->do_not_store_failed_count.fetch_add(1, std::memory_order_relaxed); } } -#endif } #endif - c.add(_length, _pagesize, _addr); - return true; // cached +#endif + return c->add(_reservation, _pagesize, _addr); } catch(...) { @@ -263,8 +319,14 @@ bool map_handle::_recycle_map() noexcept map_handle::cache_statistics map_handle::trim_cache(std::chrono::steady_clock::time_point older_than, size_t max_items) noexcept { - return detail::map_handle_cache().trim_cache(older_than, max_items); + auto *c = detail::map_handle_cache(); + return (c != nullptr) ? c->trim_cache(older_than, max_items) : cache_statistics{}; } +bool map_handle::set_cache_disabled(bool disabled) noexcept +{ + auto *c = detail::map_handle_cache(); + return (c != nullptr) ? c->set_cache_disabled(disabled) : true; +} LLFIO_V2_NAMESPACE_END diff --git a/include/llfio/v2.0/detail/impl/path_discovery.ipp b/include/llfio/v2.0/detail/impl/path_discovery.ipp index 9a71595d..f9d1e731 100644 --- a/include/llfio/v2.0/detail/impl/path_discovery.ipp +++ b/include/llfio/v2.0/detail/impl/path_discovery.ipp @@ -156,8 +156,8 @@ namespace path_discovery #if 0 fprintf(stderr, "path_discovery::verified_temporary_directories() failed to open %s due to %s\n", ps.all[n].path.path().c_str(), _h.error().message().c_str()); - path_view::c_str<> zpath(ps.all[n].path, path_view::zero_terminated); - fprintf(stderr, "path_view::c_str says buffer = %p (%s) length = %u\n", zpath.buffer, zpath.buffer, (unsigned) zpath.length); + path_view::zero_terminated_rendered_path<> zpath(ps.all[n].path); + fprintf(stderr, "path_view::c_str says buffer = %p (%s) length = %u\n", zpath.c_str(), zpath.c_str(), (unsigned) zpath.size()); visit(ps.all[n].path, [](auto _sv) { char buffer[1024]; memcpy(buffer, (const char *) _sv.data(), _sv.size()); diff --git a/include/llfio/v2.0/detail/impl/path_view.ipp b/include/llfio/v2.0/detail/impl/path_view.ipp index f47561ad..4d0f2976 100644 --- a/include/llfio/v2.0/detail/impl/path_view.ipp +++ b/include/llfio/v2.0/detail/impl/path_view.ipp @@ -98,7 +98,7 @@ namespace detail assert(std::codecvt_base::noconv != result); if(std::codecvt_base::noconv == result) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str should never do identity reencoding"); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path should never do identity reencoding"); abort(); } if(std::codecvt_base::error == result) @@ -109,7 +109,7 @@ namespace detail throw std::system_error(make_error_code(std::errc::illegal_byte_sequence)); } // Otherwise proceed anyway :) - LLFIO_LOG_WARN(nullptr, "path_view_component::c_str saw failure to completely convert input encoding"); + LLFIO_LOG_WARN(nullptr, "path_view_component::rendered_path saw failure to completely convert input encoding"); result = std::codecvt_base::ok; } if(std::codecvt_base::ok == result) @@ -139,12 +139,12 @@ namespace detail char *reencode_path_to(size_t & /*unused*/, char * /*unused*/, size_t /*unused*/, const LLFIO_V2_NAMESPACE::byte * /*unused*/, size_t /*unused*/, const std::locale * /*unused*/) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see passthrough."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see passthrough."); abort(); } char *reencode_path_to(size_t & /*unused*/, char * /*unused*/, size_t /*unused*/, const char * /*unused*/, size_t /*unused*/, const std::locale * /*unused*/) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see identity."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see identity."); abort(); } char *reencode_path_to(size_t &toallocate, char *dest_buffer, size_t dest_buffer_length, const wchar_t *src_buffer, size_t src_buffer_length, @@ -252,7 +252,7 @@ namespace detail assert(std::codecvt_base::noconv != result); if(std::codecvt_base::noconv == result) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str should never do identity reencoding"); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path should never do identity reencoding"); abort(); } if(std::codecvt_base::error == result) @@ -299,7 +299,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see identity."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see identity."); abort(); #endif } @@ -318,7 +318,7 @@ namespace detail wchar_t *reencode_path_to(size_t & /*unused*/, wchar_t * /*unused*/, size_t /*unused*/, const LLFIO_V2_NAMESPACE::byte * /*unused*/, size_t /*unused*/, const std::locale * /*unused*/) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see passthrough."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see passthrough."); abort(); } wchar_t *reencode_path_to(size_t &toallocate, wchar_t *dest_buffer, size_t dest_buffer_length, const char *src_buffer, size_t src_buffer_length, @@ -417,7 +417,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char to wchar_t conversion on libc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char to wchar_t conversion on libc++."); abort(); #elif defined(__GLIBCXX__) (void) toallocate; @@ -426,7 +426,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char to wchar_t conversion on libstdc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char to wchar_t conversion on libstdc++."); abort(); #else return _reencode_path_to(toallocate, dest_buffer, dest_buffer_length, src_buffer, src_buffer_length, loc); @@ -435,7 +435,7 @@ namespace detail wchar_t *reencode_path_to(size_t & /*unused*/, wchar_t * /*unused*/, size_t /*unused*/, const wchar_t * /*unused*/, size_t /*unused*/, const std::locale * /*unused*/) { - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see identity."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see identity."); abort(); } wchar_t *reencode_path_to(size_t &toallocate, wchar_t *dest_buffer, size_t dest_buffer_length, const char8_t *src_buffer, size_t src_buffer_length, @@ -449,7 +449,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char8_t to wchar_t conversion on libc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char8_t to wchar_t conversion on libc++."); abort(); #elif defined(__GLIBCXX__) (void) toallocate; @@ -458,7 +458,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char8_t to wchar_t conversion on libstdc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char8_t to wchar_t conversion on libstdc++."); abort(); #else return _reencode_path_to(toallocate, dest_buffer, dest_buffer_length, (const char *) src_buffer, src_buffer_length, loc); @@ -472,7 +472,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char8_t to wchar_t conversion on libc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char8_t to wchar_t conversion on libc++."); abort(); #elif defined(__GLIBCXX__) (void) toallocate; @@ -481,7 +481,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char8_t to wchar_t conversion on libstdc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char8_t to wchar_t conversion on libstdc++."); abort(); #else return _reencode_path_to(toallocate, dest_buffer, dest_buffer_length, src_buffer, src_buffer_length, loc); @@ -498,7 +498,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function should never see identity."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function should never see identity."); abort(); #elif defined(_LIBCPP_VERSION) (void) toallocate; @@ -507,7 +507,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char16_t to wchar_t conversion on libc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char16_t to wchar_t conversion on libc++."); abort(); #elif defined(__GLIBCXX__) (void) toallocate; @@ -516,7 +516,7 @@ namespace detail (void) src_buffer; (void) src_buffer_length; (void) loc; - LLFIO_LOG_FATAL(nullptr, "path_view_component::c_str reencoding function does not support char16_t to wchar_t conversion on libstdc++."); + LLFIO_LOG_FATAL(nullptr, "path_view_component::rendered_path reencoding function does not support char16_t to wchar_t conversion on libstdc++."); abort(); #else return _reencode_path_to(toallocate, dest_buffer, dest_buffer_length, src_buffer, src_buffer_length, loc); diff --git a/include/llfio/v2.0/detail/impl/posix/directory_handle.ipp b/include/llfio/v2.0/detail/impl/posix/directory_handle.ipp index d9bfc2be..81165413 100644 --- a/include/llfio/v2.0/detail/impl/posix/directory_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/directory_handle.ipp @@ -80,7 +80,7 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa // really ought to be cloning the handle. But let's humour him. path = "."; } - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); auto rename_random_dir_over_existing_dir = [_mode, _caching, flags](const path_handle &base, path_view_type path) -> result<directory_handle> { // Take a path handle to the directory containing the file auto path_parent = path.parent_path(); @@ -110,7 +110,7 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa { if(_creation == creation::if_needed || _creation == creation::only_if_not_exist || _creation == creation::always_new) { - if(-1 == ::mkdirat(base.native_handle().fd, zpath.buffer, 0x1f8 /*770*/)) + if(-1 == ::mkdirat(base.native_handle().fd, zpath.c_str(), 0x1f8 /*770*/)) { if(EEXIST != errno || _creation == creation::only_if_not_exist) { @@ -129,13 +129,13 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa } attribs &= ~(O_CREAT | O_EXCL); } - nativeh.fd = ::openat(base.native_handle().fd, zpath.buffer, attribs); + nativeh.fd = ::openat(base.native_handle().fd, zpath.c_str(), attribs); } else { if(_creation == creation::if_needed || _creation == creation::only_if_not_exist || _creation == creation::always_new) { - if(-1 == ::mkdir(zpath.buffer, 0x1f8 /*770*/)) + if(-1 == ::mkdir(zpath.c_str(), 0x1f8 /*770*/)) { if(EEXIST != errno || _creation == creation::only_if_not_exist) { @@ -154,7 +154,7 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa } attribs &= ~(O_CREAT | O_EXCL); } - nativeh.fd = ::open(zpath.buffer, attribs); + nativeh.fd = ::open(zpath.c_str(), attribs); } if(-1 == nativeh.fd) { @@ -267,13 +267,13 @@ result<directory_handle::buffers_type> directory_handle::read(io_request<buffers return std::move(req.buffers); } // Is glob a single entry match? If so, this is really a stat call - path_view_type::c_str<> zglob(req.glob, path_view::zero_terminated); + path_view_type::zero_terminated_rendered_path<> zglob(req.glob); if(!req.glob.empty() && !req.glob.contains_glob()) { struct stat s { }; - if(-1 == ::fstatat(_v.fd, zglob.buffer, &s, AT_SYMLINK_NOFOLLOW)) + if(-1 == ::fstatat(_v.fd, zglob.c_str(), &s, AT_SYMLINK_NOFOLLOW)) { return posix_error(); } @@ -443,7 +443,7 @@ result<directory_handle::buffers_type> directory_handle::read(io_request<buffers goto cont; } } - if(!req.glob.empty() && fnmatch(zglob.buffer, dent->d_name, 0) != 0) + if(!req.glob.empty() && fnmatch(zglob.c_str(), dent->d_name, 0) != 0) { goto cont; } diff --git a/include/llfio/v2.0/detail/impl/posix/file_handle.ipp b/include/llfio/v2.0/detail/impl/posix/file_handle.ipp index cf84c3db..3dab95e3 100644 --- a/include/llfio/v2.0/detail/impl/posix/file_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/file_handle.ipp @@ -1,5 +1,5 @@ /* A handle to something -(C) 2015-2020 Niall Douglas <http://www.nedproductions.biz/> (8 commits) +(C) 2015-2021 Niall Douglas <http://www.nedproductions.biz/> (8 commits) File Created: Dec 2015 @@ -51,14 +51,14 @@ result<file_handle> file_handle::file(const path_handle &base, file_handle::path OUTCOME_TRY(auto &&attribs, attribs_from_handle_mode_caching_and_flags(nativeh, _mode, _creation, _caching, flags)); attribs &= ~O_NONBLOCK; nativeh.behaviour &= ~native_handle_type::disposition::nonblocking; - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); if(base.is_valid()) { - nativeh.fd = ::openat(base.native_handle().fd, zpath.buffer, attribs, 0x1b0 /*660*/); + nativeh.fd = ::openat(base.native_handle().fd, zpath.c_str(), attribs, 0x1b0 /*660*/); } else { - nativeh.fd = ::open(zpath.buffer, attribs, 0x1b0 /*660*/); + nativeh.fd = ::open(zpath.c_str(), attribs, 0x1b0 /*660*/); } if(-1 == nativeh.fd) { @@ -536,11 +536,11 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte while(extent.length > 0) { deadline nd; - auto towrite = (extent.length < blocksize) ? (size_t) extent.length : blocksize; - buffer_type b(buffer, towrite); + const size_t towrite = (extent.length < blocksize) ? (size_t) extent.length : blocksize; + buffer_type b(buffer, utils::round_up_to_page_size(towrite, 4096) /* to allow aligned i/o files */); LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); OUTCOME_TRY(auto &&readed, read({{&b, 1}, extent.offset}, nd)); - const_buffer_type cb(readed.front()); + const_buffer_type cb(readed.front().data(), std::min(readed.front().size(), towrite)); if(cb.size() == 0) { return ret; @@ -590,8 +590,8 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte See https://github.com/openzfs/zfs/issues/11697 for more. */ - char b; - if(-1 == ::pread(_v.fd, &b, 1, end)) + alignas(4096) char b[4096]; + if(-1 == ::pread(_v.fd, b, requires_aligned_io() ? 4096 : 1, end)) { return posix_error(); } @@ -837,16 +837,17 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte buffer = utils::page_allocator<byte>().allocate(blocksize); } deadline nd; - buffer_type b(buffer, thisblock); + buffer_type b(buffer, utils::round_up_to_page_size(thisblock, 4096) /* to allow aligned i/o files */); LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); OUTCOME_TRY(auto &&readed, read({{&b, 1}, item.src.offset + thisoffset}, nd)); buffer_dirty = true; - if(readed.front().size() != thisblock) + if(readed.front().size() < thisblock) { return errc::resource_unavailable_try_again; // something is wrong } + readed.front() = {readed.front().data(), thisblock}; LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); - const_buffer_type cb(readed.front()); + const_buffer_type cb(readed.front().data(), thisblock); if(item.destination_extents_are_new) { // If we don't need to reset the bytes in the destination, try to elide diff --git a/include/llfio/v2.0/detail/impl/posix/fs_handle.ipp b/include/llfio/v2.0/detail/impl/posix/fs_handle.ipp index f59638e2..203d4dd0 100644 --- a/include/llfio/v2.0/detail/impl/posix/fs_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/fs_handle.ipp @@ -131,12 +131,12 @@ namespace detail return success(std::move(currentdirh)); } // stat the same file name, and compare dev and inode - path_view::c_str<> zpath(filename, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(filename); struct stat s { }; memset(&s, 0, sizeof(s)); - if(-1 == ::fstatat(currentdirh.native_handle().fd, zpath.buffer, &s, AT_SYMLINK_NOFOLLOW)) + if(-1 == ::fstatat(currentdirh.native_handle().fd, zpath.c_str(), &s, AT_SYMLINK_NOFOLLOW)) { continue; } @@ -191,7 +191,7 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo { LLFIO_LOG_FUNCTION_CALL(this); auto &h = const_cast<handle &>(_get_handle()); - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); #ifdef O_TMPFILE // If the handle was created with O_TMPFILE, we need a different approach if(h.flags() & handle::flag::anonymous_inode) @@ -202,7 +202,7 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo } char _path[PATH_MAX]; snprintf(_path, PATH_MAX, "/proc/self/fd/%d", h.native_handle().fd); - if(-1 == ::linkat(AT_FDCWD, _path, base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, AT_SYMLINK_FOLLOW)) + if(-1 == ::linkat(AT_FDCWD, _path, base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), AT_SYMLINK_FOLLOW)) { return posix_error(); } @@ -224,22 +224,22 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo errno = 0; if(-1 != #if defined __aarch64__ - syscall(276 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(276 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #elif defined __arm__ - syscall(382 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(382 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #elif defined __i386__ - syscall(353 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(353 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #elif defined __powerpc64__ - syscall(357 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(357 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #elif defined __sparc__ - syscall(345 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(345 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #elif defined __x86_64__ - syscall(316 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, + syscall(316 /*__NR_renameat2*/, dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 1 /*RENAME_NOREPLACE*/) #else #error Unknown Linux platform @@ -255,7 +255,7 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo #endif // Otherwise we need to use linkat followed by unlinkat, carefully reopening the file descriptor // to preserve path tracking - if(-1 == ::linkat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, 0)) + if(-1 == ::linkat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 0)) { return posix_error(); } @@ -295,7 +295,7 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo #endif | O_SYNC); #endif - int fd = ::openat(base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, attribs, 0x1b0 /*660*/); + int fd = ::openat(base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), attribs, 0x1b0 /*660*/); if(-1 == fd) { errcode = errno; @@ -319,7 +319,7 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo } return success(); } - if(-1 == ::renameat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer)) + if(-1 == ::renameat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str())) { return posix_error(); } @@ -330,10 +330,10 @@ result<void> fs_handle::link(const path_handle &base, path_view_type path, deadl { LLFIO_LOG_FUNCTION_CALL(this); auto &h = const_cast<handle &>(_get_handle()); - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); #ifdef AT_EMPTY_PATH // Try to use the fd linking syscall - if(-1 != ::linkat(h.native_handle().fd, "", base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, AT_EMPTY_PATH)) + if(-1 != ::linkat(h.native_handle().fd, "", base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), AT_EMPTY_PATH)) { return success(); } @@ -345,7 +345,7 @@ result<void> fs_handle::link(const path_handle &base, path_view_type path, deadl OUTCOME_TRY(_fetch_inode()); } OUTCOME_TRY(auto &&dirh, detail::containing_directory(std::ref(filename), h, *this, d)); - if(-1 == ::linkat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, 0)) + if(-1 == ::linkat(dirh.native_handle().fd, filename.c_str(), base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.c_str(), 0)) { return posix_error(); } diff --git a/include/llfio/v2.0/detail/impl/posix/io_handle.ipp b/include/llfio/v2.0/detail/impl/posix/io_handle.ipp index 7d2e454b..b1cc1388 100644 --- a/include/llfio/v2.0/detail/impl/posix/io_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/io_handle.ipp @@ -52,7 +52,7 @@ size_t io_handle::_do_max_buffers() const noexcept v = 1; #else long r = sysconf(_SC_IOV_MAX); - if(r == -1) + if(r < 1) { #ifdef IOV_MAX r = IOV_MAX; @@ -95,6 +95,7 @@ io_handle::io_result<io_handle::buffers_type> io_handle::_do_read(io_handle::io_ for(size_t n = 0; n < reqs.buffers.size(); n++) { assert((reinterpret_cast<uintptr_t>(iov[n].iov_base) & 511) == 0); + // Reads must have aligned length for portability assert((iov[n].iov_len & 511) == 0); } } @@ -191,7 +192,7 @@ io_handle::io_result<io_handle::const_buffers_type> io_handle::_do_write(io_hand for(size_t n = 0; n < reqs.buffers.size(); n++) { assert((reinterpret_cast<uintptr_t>(iov[n].iov_base) & 511) == 0); - assert((iov[n].iov_len & 511) == 0); + // length can be unaligned for writes } } #endif diff --git a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp index bc6bf345..845b8894 100644 --- a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp @@ -156,10 +156,10 @@ map_handle::~map_handle() auto ret = map_handle::close(); if(ret.has_error()) { - LLFIO_LOG_FATAL(_v.fd, - "map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the " - "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " - "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); + LLFIO_LOG_FATAL( + _v.fd, "FATAL: map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the " + "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " + "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); abort(); } } @@ -223,7 +223,7 @@ map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_barrier(ma deadline d) noexcept { LLFIO_LOG_FUNCTION_CALL(this); - byte *addr = _addr + reqs.offset; + byte *addr = _addr + reqs.offset + (_offset & (_pagesize - 1)); size_type bytes = 0; // Check for overflow for(const auto &req : reqs.buffers) @@ -313,17 +313,31 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, prot |= PROT_READ | PROT_WRITE; flags &= ~MAP_SHARED; flags |= MAP_PRIVATE; - nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable; + if((nativeh.behaviour & + (native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable)) != + (native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable)) + { + nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable; + } } else if(_flag & section_handle::flag::write) { prot = (_flag & section_handle::flag::write_via_syscall) ? PROT_READ : (PROT_READ | PROT_WRITE); - nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable; + if((nativeh.behaviour & + (native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable)) != + (native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable)) + { + nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable | native_handle_type::disposition::writable; + } } else if(_flag & section_handle::flag::read) { prot |= PROT_READ; - nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable; + if((nativeh.behaviour & (native_handle_type::disposition::seekable | native_handle_type::disposition::readable)) != + (native_handle_type::disposition::seekable | native_handle_type::disposition::readable)) + { + nativeh.behaviour |= native_handle_type::disposition::seekable | native_handle_type::disposition::readable; + } } if(_flag & section_handle::flag::execute) { @@ -381,19 +395,21 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, #error Do not know how to specify large/huge/super pages on this platform #endif } -// printf("mmap(%p, %u, %d, %d, %d, %u)\n", ataddr, (unsigned) bytes, prot, flags, have_backing ? section->native_handle().fd : -1, (unsigned) offset); + auto _offset = offset & ~(pagesize - 1); + auto _bytes = bytes + (offset & (pagesize - 1)); +// printf("mmap(%p, %u (%u), %d, %d, %d, %u (%u))\n", ataddr, (unsigned) _bytes, (unsigned) bytes, prot, flags, have_backing ? section->native_handle().fd : -1, (unsigned) _offset, (unsigned) offset); #ifdef MAP_SYNC // Linux kernel 4.15 or later only // If backed by a file into persistent shared memory, ask the kernel to use persistent memory safe semantics if(have_backing && (_flag & section_handle::flag::nvram) && (flags & MAP_SHARED) != 0) { int flagscopy = flags & ~MAP_SHARED; flagscopy |= MAP_SHARED_VALIDATE | MAP_SYNC; - addr = ::mmap(ataddr, bytes, prot, flagscopy, fd_to_use, offset); + addr = ::mmap(ataddr, _bytes, prot, flagscopy, fd_to_use, _offset); } #endif if(addr == nullptr) { - addr = ::mmap(ataddr, bytes, prot, flags, fd_to_use, offset); + addr = ::mmap(ataddr, _bytes, prot, flags, fd_to_use, _offset); } // printf("%d mmap %p-%p\n", getpid(), addr, (char *) addr+bytes); if(MAP_FAILED == addr) // NOLINT @@ -407,7 +423,7 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, // for committed memory first and then decommit it. Which can potentially // blow out the commit limit for really large reservations, // but this is literally the only game in town on Mac OS. - if(-1 == ::madvise(addr, bytes, MADV_FREE_REUSABLE)) + if(-1 == ::madvise(addr, _bytes, MADV_FREE_REUSABLE)) { return posix_error(); } @@ -417,7 +433,7 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, if(have_backing && ((flags & map_handle::flag::disable_prefetching) || (flags & map_handle::flag::maximum_prefetching))) { int advice = (flags & map_handle::flag::disable_prefetching) ? MADV_RANDOM : MADV_SEQUENTIAL; - if(-1 == ::madvise(addr, bytes, advice)) + if(-1 == ::madvise(addr, _bytes, advice)) return posix_error(); } #endif @@ -462,9 +478,21 @@ result<map_handle> map_handle::_new_map(size_type bytes, bool fallback, section_ result<map_handle> map_handle::map(section_handle §ion, size_type bytes, extent_type offset, section_handle::flag _flag) noexcept { OUTCOME_TRY(auto &&length, section.length()); // length of the backing file + if(length <= offset) + { + length = 0; + } + else + { + length -= offset; + } if(bytes == 0u) { - bytes = length - offset; + bytes = length; + } + else if(length > bytes) + { + length = bytes; } result<map_handle> ret{map_handle(§ion, _flag)}; native_handle_type &nativeh = ret.value()._v; @@ -473,7 +501,7 @@ result<map_handle> map_handle::map(section_handle §ion, size_type bytes, ext ret.value()._addr = static_cast<byte *>(addr); ret.value()._offset = offset; ret.value()._reservation = utils::round_up_to_page_size(bytes, pagesize); - ret.value()._length = (length - offset < bytes) ? (length - offset) : bytes; // length of backing, not reservation + ret.value()._length = length; ret.value()._pagesize = pagesize; // Make my handle borrow the native handle of my backing storage ret.value()._v.fd = section.native_handle().fd; @@ -490,6 +518,18 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi if(_section != nullptr) { OUTCOME_TRY(length, _section->length()); // length of the backing file + if(length <= _offset) + { + length = 0; + } + else + { + length -= _offset; + } + if(length > _reservation) + { + length = _reservation; + } } auto _newsize = utils::round_up_to_page_size(newsize, _pagesize); if(_newsize == _reservation) @@ -518,7 +558,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi OUTCOME_TRY(auto &&addr, do_mmap(_v, nullptr, 0, _section, _pagesize, newsize, _offset, _flag)); _addr = static_cast<byte *>(addr); _reservation = _newsize; - _length = (length - _offset < newsize) ? (length - _offset) : newsize; // length of backing, not reservation + _length = length; return newsize; } #ifdef __linux__ @@ -530,7 +570,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi } _addr = static_cast<byte *>(newaddr); _reservation = _newsize; - _length = (length - _offset < newsize) ? (length - _offset) : newsize; // length of backing, not reservation + _length = length; return newsize; #else (void) permit_relocation; @@ -543,7 +583,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi extent_type offset = _offset + _reservation; OUTCOME_TRY(auto &&addr, do_mmap(_v, addrafter, MAP_FIXED | MAP_EXCL, _section, _pagesize, bytes, offset, _flag)); _reservation = _newsize; - _length = (length - _offset < newsize) ? (length - _offset) : newsize; // length of backing, not reservation + _length = length; return newsize; #else // generic POSIX, inefficient byte *addrafter = _addr + _reservation; @@ -556,7 +596,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi return errc::not_enough_memory; } _reservation = _newsize; - _length = (length - _offset < newsize) ? (length - _offset) : newsize; // length of backing, not reservation + _length = length; return newsize; #endif } @@ -566,7 +606,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi return posix_error(); } _reservation = newsize; - _length = (length - _offset < newsize) ? (length - _offset) : newsize; + _length = length; return newsize; #endif } @@ -688,7 +728,7 @@ result<map_handle::buffer_type> map_handle::do_not_store(buffer_type region) noe map_handle::io_result<map_handle::buffers_type> map_handle::_do_read(io_request<buffers_type> reqs, deadline /*d*/) noexcept { LLFIO_LOG_FUNCTION_CALL(this); - byte *addr = _addr + reqs.offset; + byte *addr = _addr + reqs.offset + (_offset & (_pagesize - 1)); size_type togo = reqs.offset < _length ? static_cast<size_type>(_length - reqs.offset) : 0; for(size_t i = 0; i < reqs.buffers.size(); i++) { @@ -711,21 +751,23 @@ map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_write(io_r LLFIO_LOG_FUNCTION_CALL(this); if(!!(_flag & section_handle::flag::write_via_syscall) && _section != nullptr && _section->backing() != nullptr) { + reqs.offset += _offset; auto r = _section->backing()->write(reqs, d); if(!r) { return std::move(r).error(); } + reqs.offset += _offset; if(reqs.offset + r.bytes_transferred() > _length) { OUTCOME_TRY(update_map()); } return std::move(r).value(); } - byte *addr = _addr + reqs.offset; + byte *addr = _addr + reqs.offset + (_offset & (_pagesize - 1)); size_type togo = reqs.offset < _length ? static_cast<size_type>(_length - reqs.offset) : 0; if(QUICKCPPLIB_NAMESPACE::signal_guard::signal_guard( - QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::undefined_memory_access, + QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::undefined_memory_access | QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::segmentation_fault, [&] { for(size_t i = 0; i < reqs.buffers.size(); i++) { diff --git a/include/llfio/v2.0/detail/impl/posix/mapped_file_handle.ipp b/include/llfio/v2.0/detail/impl/posix/mapped_file_handle.ipp index 5543b980..5f57f10b 100644 --- a/include/llfio/v2.0/detail/impl/posix/mapped_file_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/mapped_file_handle.ipp @@ -64,7 +64,7 @@ result<mapped_file_handle::size_type> mapped_file_handle::_reserve(extent_type & mapflags |= section_handle::flag::write; } OUTCOME_TRYV(_mh.close()); - OUTCOME_TRY(auto &&mh, map_handle::map(_sh, reservation, 0, mapflags)); + OUTCOME_TRY(auto &&mh, map_handle::map(_sh, reservation, _offset, mapflags)); _mh = std::move(mh); _reservation = reservation; return _reservation; diff --git a/include/llfio/v2.0/detail/impl/posix/path_handle.ipp b/include/llfio/v2.0/detail/impl/posix/path_handle.ipp index 47b242f5..582ef50c 100644 --- a/include/llfio/v2.0/detail/impl/posix/path_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/path_handle.ipp @@ -32,8 +32,8 @@ result<bool> path_handle::exists(path_view_type path) const noexcept { try { LLFIO_LOG_FUNCTION_CALL(this); - path_view::c_str<> zpath(path, path_view::zero_terminated); - int x = ::faccessat(native_handle().fd, zpath.buffer, F_OK, AT_SYMLINK_NOFOLLOW); + path_view::zero_terminated_rendered_path<> zpath(path); + int x = ::faccessat(native_handle().fd, zpath.c_str(), F_OK, AT_SYMLINK_NOFOLLOW); if(x < 0) { auto ret = posix_error(); @@ -66,14 +66,14 @@ result<path_handle> path_handle::path(const path_handle &base, path_handle::path // Linux provides this extension opening a super light weight fd to just an anchor on the filing system attribs |= O_PATH; #endif - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); if(base.is_valid()) { - nativeh.fd = ::openat(base.native_handle().fd, zpath.buffer, attribs); + nativeh.fd = ::openat(base.native_handle().fd, zpath.c_str(), attribs); } else { - nativeh.fd = ::open(zpath.buffer, attribs); + nativeh.fd = ::open(zpath.c_str(), attribs); } if(-1 == nativeh.fd) { diff --git a/include/llfio/v2.0/detail/impl/posix/process_handle.ipp b/include/llfio/v2.0/detail/impl/posix/process_handle.ipp index 63070099..2ca20c27 100644 --- a/include/llfio/v2.0/detail/impl/posix/process_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/process_handle.ipp @@ -243,16 +243,16 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<process_handle> process_handle::launch_pr return posix_error(); } - using small_path_view_c_str = path_view::c_str<filesystem::path::value_type, std::default_delete<filesystem::path::value_type[]>, 1>; + using small_path_view_c_str = path_view::zero_terminated_rendered_path<filesystem::path::value_type, std::default_delete<filesystem::path::value_type[]>, 1>; std::vector<const char *> argptrs(args.size() + 2); std::vector<small_path_view_c_str> _args; _args.reserve(args.size() + 1); - _args.emplace_back(path, path_view::zero_terminated); - argptrs[0] = _args[0].buffer; + _args.emplace_back(path); + argptrs[0] = _args[0].c_str(); for(size_t n = 0; n < args.size(); ++n) { - _args.emplace_back(args[n], path_view::zero_terminated); - argptrs[n + 1] = _args[n + 1].buffer; + _args.emplace_back(args[n]); + argptrs[n + 1] = _args[n + 1].c_str(); } std::vector<small_path_view_c_str> _envs; std::vector<const char *> envptrs; @@ -260,8 +260,8 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<process_handle> process_handle::launch_pr envptrs.reserve(env.size() + 1); for(const auto &i : env) { - _envs.emplace_back(i, path_view::zero_terminated); - envptrs.push_back(_envs.back().buffer); + _envs.emplace_back(i); + envptrs.push_back(_envs.back().c_str()); } envptrs.push_back(nullptr); #if 0 diff --git a/include/llfio/v2.0/detail/impl/posix/symlink_handle.ipp b/include/llfio/v2.0/detail/impl/posix/symlink_handle.ipp index fad232ec..9ae1ff61 100644 --- a/include/llfio/v2.0/detail/impl/posix/symlink_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/symlink_handle.ipp @@ -63,7 +63,7 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<void> symlink_handle::_create_symlink(con end_utc = d.to_time_point(); } } - path_view::c_str<> zpath(target, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(target); try { if(atomic_replace) @@ -74,8 +74,8 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<void> symlink_handle::_create_symlink(con { auto randomname = utils::random_string(32); randomname.append(".random"); - // std::cerr << "symlinkat " << zpath.buffer << " " << dirh.native_handle().fd << " " << randomname << std::endl; - if(-1 == ::symlinkat(zpath.buffer, dirh.is_valid() ? dirh.native_handle().fd : AT_FDCWD, randomname.c_str())) + // std::cerr << "symlinkat " << zpath.c_str() << " " << dirh.native_handle().fd << " " << randomname << std::endl; + if(-1 == ::symlinkat(zpath.c_str(), dirh.is_valid() ? dirh.native_handle().fd : AT_FDCWD, randomname.c_str())) { if(EEXIST == errno) { @@ -112,8 +112,8 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<void> symlink_handle::_create_symlink(con } else { - // std::cerr << "symlinkat " << zpath.buffer << " " << dirh.native_handle().fd << " " << filename << std::endl; - if(-1 == ::symlinkat(zpath.buffer, dirh.is_valid() ? dirh.native_handle().fd : AT_FDCWD, filename.c_str())) + // std::cerr << "symlinkat " << zpath.c_str() << " " << dirh.native_handle().fd << " " << filename << std::endl; + if(-1 == ::symlinkat(zpath.c_str(), dirh.is_valid() ? dirh.native_handle().fd : AT_FDCWD, filename.c_str())) { if(exists_is_ok && EEXIST == errno) { diff --git a/include/llfio/v2.0/detail/impl/posix/utils.ipp b/include/llfio/v2.0/detail/impl/posix/utils.ipp index 2332ae91..fe3aa5d2 100644 --- a/include/llfio/v2.0/detail/impl/posix/utils.ipp +++ b/include/llfio/v2.0/detail/impl/posix/utils.ipp @@ -333,7 +333,7 @@ namespace utils (values are in pages) - /proc/[pid]/smaps_rollup: + /proc/[pid]/smaps_rollup (Linux 3.16 onwards only): total_address_space_in_use = ??? MISSING total_address_space_paged_in = ??? MISSING @@ -341,147 +341,187 @@ namespace utils private_paged_in = ??? MISSING */ - if(want & process_memory_usage::want::private_committed_inaccurate) + process_memory_usage ret; + if(!!(want & process_memory_usage::want::this_process)) { - process_memory_usage ret; - if((want & process_memory_usage::want::total_address_space_in_use) || (want & process_memory_usage::want::total_address_space_paged_in) || - (want & process_memory_usage::want::private_paged_in)) + if(!(want & process_memory_usage::want::private_committed) || (want & process_memory_usage::want::private_committed_inaccurate)) { - std::vector<char> buffer(256); - OUTCOME_TRY(fill_buffer(buffer, "/proc/self/statm")); - if(buffer.size() > 1) + if((want & process_memory_usage::want::total_address_space_in_use) || (want & process_memory_usage::want::total_address_space_paged_in) || + (want & process_memory_usage::want::private_paged_in)) { - size_t file_and_shared_pages_paged_in = 0; - sscanf(buffer.data(), "%zu %zu %zu", &ret.total_address_space_in_use, &ret.total_address_space_paged_in, &file_and_shared_pages_paged_in); - ret.private_paged_in = ret.total_address_space_paged_in - file_and_shared_pages_paged_in; - ret.total_address_space_in_use *= page_size(); - ret.total_address_space_paged_in *= page_size(); - ret.private_paged_in *= page_size(); - //std::cout << string_view(buffer.data(), buffer.size()) << std::endl; + std::vector<char> buffer(256); + OUTCOME_TRY(fill_buffer(buffer, "/proc/self/statm")); + if(buffer.size() > 1) + { + size_t file_and_shared_pages_paged_in = 0; + sscanf(buffer.data(), "%zu %zu %zu", &ret.total_address_space_in_use, &ret.total_address_space_paged_in, &file_and_shared_pages_paged_in); + ret.private_paged_in = ret.total_address_space_paged_in - file_and_shared_pages_paged_in; + ret.total_address_space_in_use *= page_size(); + ret.total_address_space_paged_in *= page_size(); + ret.private_paged_in *= page_size(); + // std::cout << string_view(buffer.data(), buffer.size()) << std::endl; + } } - } - if(want & process_memory_usage::want::private_committed) - { - std::vector<char> smaps_rollup(256), maps(65536); - OUTCOME_TRY(fill_buffer(smaps_rollup, "/proc/self/smaps_rollup")); - OUTCOME_TRY(fill_buffer(maps, "/proc/self/maps")); - uint64_t lazyfree = 0; + if(want & process_memory_usage::want::private_committed) { - string_view i(smaps_rollup.data(), smaps_rollup.size()); - OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:")); + std::vector<char> smaps_rollup(256), maps(65536); + auto r = fill_buffer(smaps_rollup, "/proc/self/smaps_rollup"); + if(!r) + { + if(r.error() == errc::no_such_file_or_directory) + { + // Linux kernel is too old + return errc::operation_not_supported; + } + return std::move(r).error(); + } + OUTCOME_TRY(fill_buffer(maps, "/proc/self/maps")); + uint64_t lazyfree = 0; + { + string_view i(smaps_rollup.data(), smaps_rollup.size()); + OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:")); + } + string_view i(maps.data(), maps.size()); + size_t anonymous = 0; + for(size_t idx = 0;;) + { + idx = i.find("\n", idx); + if(idx == i.npos) + { + break; + } + idx++; + size_t start = 0, end = 0, inode = 1; + char read = 0, write = 0, executable = 0, private_ = 0; + sscanf(i.data() + idx, "%zx-%zx %c%c%c%c %*u %*u:%*u %zd", &start, &end, &read, &write, &executable, &private_, &inode); + if(inode == 0 && read == 'r' && write == 'w' && executable == '-' && private_ == 'p') + { + anonymous += end - start; + // std::cout << (end - start) << " " << i.substr(idx, 40) << std::endl; + } + } + if(lazyfree != (uint64_t) -1) + { + anonymous -= (size_t) lazyfree; + } + ret.private_committed = anonymous; } - string_view i(maps.data(), maps.size()); - size_t anonymous = 0; - for(size_t idx = 0;;) + } + else + { + std::vector<char> buffer(1024 * 1024); + OUTCOME_TRY(fill_buffer(buffer, "/proc/self/smaps")); + const string_view totalview(buffer.data(), buffer.size()); + // std::cerr << totalview << std::endl; + std::vector<string_view> anon_entries, non_anon_entries; + anon_entries.reserve(32); + non_anon_entries.reserve(32); + auto find_item = [&](size_t idx) -> string_view { + auto x = totalview.rfind("\nSize:", idx); + if(x == string_view::npos) + { + return {}; + } + x = totalview.rfind("\n", x - 1); + if(x == string_view::npos) + { + x = 0; + } + else + { + x++; + } + return totalview.substr(x, idx - x); + }; + for(string_view item = find_item(totalview.size()); item != string_view(); item = find_item(item.data() - totalview.data())) { - idx = i.find("\n", idx); - if(idx == i.npos) + // std::cout << "***" << item << "***"; + // hexaddr-hexaddr flags offset dev:id inode [path] + size_t inode = 1; + sscanf(item.data(), "%*x-%*x %*c%*c%*c%*c %*x %*c%*c:%*c%*c %zu", &inode); + auto vmflagsidx = item.rfind("\nVmFlags:"); + if(vmflagsidx == string_view::npos) { - break; + return errc::illegal_byte_sequence; + } + // Is there " ac" after vmflagsidx? + if(string_view::npos != item.find(" ac", vmflagsidx) && inode == 0) + { + // std::cerr << "Adding anon entry at offset " << itemtopidx << std::endl; + anon_entries.push_back(item); } - idx++; - size_t start = 0, end = 0, inode = 1; - char read = 0, write = 0, executable = 0, private_ = 0; - sscanf(i.data() + idx, "%zx-%zx %c%c%c%c %*u %*u:%*u %zd", &start, &end, &read, &write, &executable, &private_, &inode); - if(inode == 0 && read == 'r' && write == 'w' && executable == '-' && private_ == 'p') + else { - anonymous += end - start; - // std::cout << (end - start) << " " << i.substr(idx, 40) << std::endl; + // std::cerr << "Adding non-anon entry at offset " << itemtopidx << std::endl; + non_anon_entries.push_back(item); } } - if(lazyfree != (uint64_t) -1) + // std::cerr << "Anon entries:"; + for(auto &i : anon_entries) { - anonymous -= (size_t) lazyfree; + OUTCOME_TRY(auto &&size, parse(i, "\nSize:")); // amount committed + OUTCOME_TRY(auto &&rss, parse(i, "\nRss:")); // amount paged in + OUTCOME_TRY(auto &&anonymous, parse(i, "\nAnonymous:")); // amount actually dirtied + OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:")); // amount "decommitted" on Linux to avoid a VMA split + if(size != (uint64_t) -1 && rss != (uint64_t) -1 && anonymous != (uint64_t) -1) + { + ret.total_address_space_in_use += size; + ret.total_address_space_paged_in += rss; + ret.private_committed += size; + if(lazyfree != (uint64_t) -1) + { + ret.total_address_space_paged_in -= lazyfree; + ret.private_committed -= lazyfree; + } + ret.private_paged_in += rss; + } + // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; } - ret.private_committed = anonymous; - } - return ret; - } - std::vector<char> buffer(1024 * 1024); - OUTCOME_TRY(fill_buffer(buffer, "/proc/self/smaps")); - const string_view totalview(buffer.data(), buffer.size()); - // std::cerr << totalview << std::endl; - std::vector<string_view> anon_entries, non_anon_entries; - anon_entries.reserve(32); - non_anon_entries.reserve(32); - auto find_item = [&](size_t idx) -> string_view { - auto x = totalview.rfind("\nSize:", idx); - if(x == string_view::npos) - { - return {}; - } - x = totalview.rfind("\n", x - 1); - if(x == string_view::npos) - { - x = 0; - } - else - { - x++; - } - return totalview.substr(x, idx - x); - }; - for(string_view item = find_item(totalview.size()); item != string_view(); item = find_item(item.data() - totalview.data())) - { - // std::cout << "***" << item << "***"; - // hexaddr-hexaddr flags offset dev:id inode [path] - size_t inode = 1; - sscanf(item.data(), "%*x-%*x %*c%*c%*c%*c %*x %*c%*c:%*c%*c %zu", &inode); - auto vmflagsidx = item.rfind("\nVmFlags:"); - if(vmflagsidx == string_view::npos) - { - return errc::illegal_byte_sequence; - } - // Is there " ac" after vmflagsidx? - if(string_view::npos != item.find(" ac", vmflagsidx) && inode == 0) - { - // std::cerr << "Adding anon entry at offset " << itemtopidx << std::endl; - anon_entries.push_back(item); - } - else - { - // std::cerr << "Adding non-anon entry at offset " << itemtopidx << std::endl; - non_anon_entries.push_back(item); - } - } - process_memory_usage ret; - // std::cerr << "Anon entries:"; - for(auto &i : anon_entries) - { - OUTCOME_TRY(auto &&size, parse(i, "\nSize:")); - OUTCOME_TRY(auto &&rss, parse(i, "\nRss:")); - OUTCOME_TRY(auto &&anonymous, parse(i, "\nAnonymous:")); - OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:")); - if(size != (uint64_t) -1 && rss != (uint64_t) -1 && anonymous != (uint64_t) -1) - { - ret.total_address_space_in_use += size; - ret.total_address_space_paged_in += rss; - ret.private_committed += anonymous; - if(lazyfree != (uint64_t) -1) + // std::cerr << "\n\nNon-anon entries:"; + for(auto &i : non_anon_entries) { - ret.total_address_space_paged_in -= lazyfree; - ret.private_committed -= lazyfree; + OUTCOME_TRY(auto &&size, parse(i, "\nSize:")); + OUTCOME_TRY(auto &&rss, parse(i, "\nRss:")); + OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:")); + if(size != (uint64_t) -1 && rss != (uint64_t) -1) + { + ret.total_address_space_in_use += size; + ret.total_address_space_paged_in += rss; + if(lazyfree != (uint64_t) -1) + { + ret.total_address_space_in_use -= lazyfree; + } + } + // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; } - ret.private_paged_in += rss; } - // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; } - // std::cerr << "\n\nNon-anon entries:"; - for(auto &i : non_anon_entries) + if(!!(want & process_memory_usage::want::this_system)) { - OUTCOME_TRY(auto &&size, parse(i, "\nSize:")); - OUTCOME_TRY(auto &&rss, parse(i, "\nRss:")); - OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:")); - if(size != (uint64_t) -1 && rss != (uint64_t) -1) + std::vector<char> buffer(1024); + OUTCOME_TRY(fill_buffer(buffer, "/proc/meminfo")); + if(buffer.size() > 1) { - ret.total_address_space_in_use += size; - ret.total_address_space_paged_in += rss; - if(lazyfree != (uint64_t) -1) + string_view i(buffer.data(), buffer.size()); + OUTCOME_TRY(ret.system_physical_memory_total, parse(i, "MemTotal:")); + OUTCOME_TRY(ret.system_physical_memory_available, parse(i, "\nMemAvailable:")); + if((uint64_t) -1 == ret.system_physical_memory_available) { - ret.total_address_space_in_use -= lazyfree; + // MemAvailable is >= Linux 3.14, so let's approximate what it would be + OUTCOME_TRY(auto &&memfree, parse(i, "\nMemFree:")); + OUTCOME_TRY(auto &&cached, parse(i, "\nCached:")); + OUTCOME_TRY(auto &&swapcached, parse(i, "\nSwapCached:")); + ret.system_physical_memory_available = memfree + cached + swapcached; } + OUTCOME_TRY(ret.system_commit_charge_maximum, parse(i, "\nCommitLimit:")); + OUTCOME_TRY(ret.system_commit_charge_available, parse(i, "\nCommitted_AS:")); + OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:")); + if(lazyfree == (uint64_t) -1) + { + lazyfree = 0; + } + ret.system_commit_charge_available = ret.system_commit_charge_maximum - ret.system_commit_charge_available + lazyfree; } - // std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl; } return ret; } @@ -493,29 +533,48 @@ namespace utils (void) want; kern_return_t error; mach_msg_type_number_t outCount; - task_vm_info_data_t vmInfo; - // task_kernelmemory_info_data_t kmInfo; + process_memory_usage ret; + if(!!(want & process_memory_usage::want::this_process)) + { + task_vm_info_data_t vmInfo; + // task_kernelmemory_info_data_t kmInfo; - outCount = TASK_VM_INFO_COUNT; - error = task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &vmInfo, &outCount); - if(error != KERN_SUCCESS) + outCount = TASK_VM_INFO_COUNT; + error = task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &vmInfo, &outCount); + if(error != KERN_SUCCESS) + { + return errc::invalid_argument; + } + // outCount = TASK_KERNELMEMORY_INFO_COUNT; + // error = task_info(mach_task_self(), TASK_KERNELMEMORY_INFO, (task_info_t)&kmInfo, &outCount); + // if (error != KERN_SUCCESS) { + // return errc::invalid_argument; + //} + // std::cout << vmInfo.virtual_size << "\n" << vmInfo.region_count << "\n" << vmInfo.resident_size << "\n" << vmInfo.device << "\n" << vmInfo.internal << + // "\n" << vmInfo.external << "\n" << vmInfo.reusable << "\n" << vmInfo.purgeable_volatile_pmap<< "\n" << vmInfo.purgeable_volatile_resident << "\n" << + // vmInfo.purgeable_volatile_virtual << "\n" << vmInfo.compressed << "\n" << vmInfo.phys_footprint << std::endl; std::cout << "\n" << kmInfo.total_palloc + // << + // "\n" << kmInfo.total_pfree << "\n" << kmInfo.total_salloc << "\n" << kmInfo.total_sfree << std::endl; + ret.total_address_space_in_use = vmInfo.virtual_size; + ret.total_address_space_paged_in = vmInfo.resident_size; + ret.private_committed = vmInfo.internal + vmInfo.compressed; + ret.private_paged_in = vmInfo.phys_footprint; + } + if(!!(want & process_memory_usage::want::this_system)) { - return errc::invalid_argument; + vm_statistics_data_t vmStat; + outCount = HOST_VM_INFO_COUNT; + error = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vmStat, &outCount); + if(error != KERN_SUCCESS) + { + return errc::invalid_argument; + } + ret.system_physical_memory_total = ((uint64_t) vmStat.free_count + vmStat.active_count + vmStat.inactive_count + vmStat.wire_count) * page_size(); + ret.system_physical_memory_available = ((uint64_t) vmStat.free_count + vmStat.inactive_count) * page_size(); + // Not sure how to retrieve these on Mac OS + // ret.system_commit_charge_maximum = (uint64_t) pi.CommitLimit * page_size(); + // ret.system_commit_charge_available = (uint64_t) pi.CommitTotal * page_size(); } - // outCount = TASK_KERNELMEMORY_INFO_COUNT; - // error = task_info(mach_task_self(), TASK_KERNELMEMORY_INFO, (task_info_t)&kmInfo, &outCount); - // if (error != KERN_SUCCESS) { - // return errc::invalid_argument; - //} - // std::cout << vmInfo.virtual_size << "\n" << vmInfo.region_count << "\n" << vmInfo.resident_size << "\n" << vmInfo.device << "\n" << vmInfo.internal << - // "\n" << vmInfo.external << "\n" << vmInfo.reusable << "\n" << vmInfo.purgeable_volatile_pmap<< "\n" << vmInfo.purgeable_volatile_resident << "\n" << - // vmInfo.purgeable_volatile_virtual << "\n" << vmInfo.compressed << "\n" << vmInfo.phys_footprint << std::endl; std::cout << "\n" << kmInfo.total_palloc << - // "\n" << kmInfo.total_pfree << "\n" << kmInfo.total_salloc << "\n" << kmInfo.total_sfree << std::endl; - process_memory_usage ret; - ret.total_address_space_in_use = vmInfo.virtual_size; - ret.total_address_space_paged_in = vmInfo.resident_size; - ret.private_committed = vmInfo.internal + vmInfo.compressed; - ret.private_paged_in = vmInfo.phys_footprint; return ret; #else #error Unknown platform diff --git a/include/llfio/v2.0/detail/impl/reduce.ipp b/include/llfio/v2.0/detail/impl/reduce.ipp index 1ed837cd..3e70b072 100644 --- a/include/llfio/v2.0/detail/impl/reduce.ipp +++ b/include/llfio/v2.0/detail/impl/reduce.ipp @@ -65,10 +65,10 @@ namespace algorithm const DWORD deletedir_ntflags = 0x20 /*FILE_SYNCHRONOUS_IO_NONALERT*/ | 0x00200000 /*FILE_OPEN_REPARSE_POINT*/ | 0x00001000 /*FILE_DELETE_ON_CLOSE*/ | 0x01 /*FILE_DIRECTORY_FILE*/; IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(leafname, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(leafname); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); OBJECT_ATTRIBUTES oa{}; memset(&oa, 0, sizeof(oa)); @@ -140,9 +140,9 @@ namespace algorithm } return ntkernel_error(ntstat); #else - path_view::c_str<> zpath(leafname, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(leafname); errno = 0; - if(is_dir || -1 == ::unlinkat(dirh.native_handle().fd, zpath.buffer, 0)) + if(is_dir || -1 == ::unlinkat(dirh.native_handle().fd, zpath.data(), 0)) { if(ENOENT == errno) { @@ -152,7 +152,7 @@ namespace algorithm if(is_dir || EISDIR == errno) { // Try to remove it as a directory, if it's empty we've saved a recurse - if(-1 != ::unlinkat(dirh.native_handle().fd, zpath.buffer, AT_REMOVEDIR)) + if(-1 != ::unlinkat(dirh.native_handle().fd, zpath.data(), AT_REMOVEDIR)) { // std::cout << "Removed quickly " << (dirh.current_path().value() / entry.leafname.path()) << std::endl; return success(); @@ -173,10 +173,10 @@ namespace algorithm const DWORD renamefile_ntflags = 0x20 /*FILE_SYNCHRONOUS_IO_NONALERT*/ | 0x00200000 /*FILE_OPEN_REPARSE_POINT*/ | 0x040 /*FILE_NON_DIRECTORY_FILE*/; const DWORD renamedir_ntflags = 0x20 /*FILE_SYNCHRONOUS_IO_NONALERT*/ | 0x00200000 /*FILE_OPEN_REPARSE_POINT*/ | 0x01 /*FILE_DIRECTORY_FILE*/; IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(leafname, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(leafname); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); OBJECT_ATTRIBUTES oa{}; memset(&oa, 0, sizeof(oa)); @@ -245,12 +245,12 @@ namespace algorithm return success(); #else (void) is_dir; - path_view::c_str<> zpath(leafname, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(leafname); if(dirh.unique_id() != topdirh.unique_id()) { // Try renaming it into topdirh auto randomname = utils::random_string(32); - if(-1 != ::renameat(topdirh.native_handle().fd, randomname.c_str(), dirh.native_handle().fd, zpath.buffer)) + if(-1 != ::renameat(topdirh.native_handle().fd, randomname.c_str(), dirh.native_handle().fd, zpath.data())) { return success(); } diff --git a/include/llfio/v2.0/detail/impl/safe_byte_ranges.ipp b/include/llfio/v2.0/detail/impl/safe_byte_ranges.ipp index 17eaf0fb..764afed1 100644 --- a/include/llfio/v2.0/detail/impl/safe_byte_ranges.ipp +++ b/include/llfio/v2.0/detail/impl/safe_byte_ranges.ipp @@ -379,11 +379,11 @@ namespace algorithm { try { - path_view::c_str<> zpath(lockfile, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(lockfile); struct stat s { }; - if(-1 == ::fstatat(base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.buffer, &s, AT_SYMLINK_NOFOLLOW)) + if(-1 == ::fstatat(base.is_valid() ? base.native_handle().fd : AT_FDCWD, zpath.data(), &s, AT_SYMLINK_NOFOLLOW)) { return posix_error(); } diff --git a/include/llfio/v2.0/detail/impl/traverse.ipp b/include/llfio/v2.0/detail/impl/traverse.ipp index 7c4be156..dccfffcd 100644 --- a/include/llfio/v2.0/detail/impl/traverse.ipp +++ b/include/llfio/v2.0/detail/impl/traverse.ipp @@ -261,8 +261,8 @@ namespace algorithm { struct ::stat stat; memset(&stat, 0, sizeof(stat)); - path_view::c_str<> zpath(entry.leafname, path_view::zero_terminated); - if(::fstatat(mydirh->native_handle().fd, zpath.buffer, &stat, AT_SYMLINK_NOFOLLOW) >= 0) + path_view::zero_terminated_rendered_path<> zpath(entry.leafname); + if(::fstatat(mydirh->native_handle().fd, zpath.data(), &stat, AT_SYMLINK_NOFOLLOW) >= 0) { entry.stat.st_type = [](uint16_t mode) { switch(mode & S_IFMT) diff --git a/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp b/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp index f8719e0c..4df473c6 100644 --- a/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp @@ -89,11 +89,11 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa ntflags |= 0x01 /*FILE_DIRECTORY_FILE*/; // required to open a directory IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -164,8 +164,8 @@ result<directory_handle> directory_handle::directory(const path_handle &base, pa break; } attribs |= FILE_FLAG_BACKUP_SEMANTICS; // required to open a directory - path_view::c_str<> zpath(path, path_view::zero_terminated); - if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.buffer, access, fileshare, nullptr, creation, attribs, nullptr, true))) // NOLINT + path_view::zero_terminated_rendered_path<> zpath(path); + if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.data(), access, fileshare, nullptr, creation, attribs, nullptr, true))) // NOLINT { DWORD errcode = GetLastError(); if(creation::always_new == _creation && 0xb7 /*ERROR_ALREADY_EXISTS*/ == errcode) @@ -303,11 +303,11 @@ result<directory_handle::buffers_type> directory_handle::read(io_request<buffers } UNICODE_STRING _glob{}; memset(&_glob, 0, sizeof(_glob)); - path_view_type::c_str<> zglob(req.glob, path_view::not_zero_terminated); + path_view_type::not_zero_terminated_rendered_path<> zglob(req.glob); if(!req.glob.empty()) { - _glob.Buffer = const_cast<wchar_t *>(zglob.buffer); - _glob.Length = (USHORT)(zglob.length * sizeof(wchar_t)); + _glob.Buffer = const_cast<wchar_t *>(zglob.data()); + _glob.Length = (USHORT)(zglob.size() * sizeof(wchar_t)); _glob.MaximumLength = _glob.Length + sizeof(wchar_t); } what_to_enumerate_type *buffer = nullptr; diff --git a/include/llfio/v2.0/detail/impl/windows/file_handle.ipp b/include/llfio/v2.0/detail/impl/windows/file_handle.ipp index 5b36810f..d3417b61 100644 --- a/include/llfio/v2.0/detail/impl/windows/file_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/file_handle.ipp @@ -1,5 +1,5 @@ /* A handle to a file -(C) 2015-2020 Niall Douglas <http://www.nedproductions.biz/> (16 commits) +(C) 2015-2021 Niall Douglas <http://www.nedproductions.biz/> (16 commits) File Created: Dec 2015 @@ -71,11 +71,11 @@ result<file_handle> file_handle::file(const path_handle &base, file_handle::path ntflags |= 0x040 /*FILE_NON_DIRECTORY_FILE*/; // do not open a directory IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -139,8 +139,8 @@ result<file_handle> file_handle::file(const path_handle &base, file_handle::path creation = CREATE_ALWAYS; break; } - path_view::c_str<> zpath(path, path_view::zero_terminated); - if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.buffer, access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT + path_view::zero_terminated_rendered_path<> zpath(path); + if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.data(), access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT { DWORD errcode = GetLastError(); // assert(false); @@ -461,11 +461,11 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte while(extent.length > 0) { deadline nd; - auto towrite = (extent.length < blocksize) ? (size_t) extent.length : blocksize; - buffer_type b(buffer, towrite); + const size_t towrite = (extent.length < blocksize) ? (size_t) extent.length : blocksize; + buffer_type b(buffer, utils::round_up_to_page_size(towrite, 4096) /* to allow aligned i/o files */); LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); OUTCOME_TRY(auto &&readed, read({{&b, 1}, extent.offset}, nd)); - const_buffer_type cb(readed.front()); + const_buffer_type cb(readed.front().data(), std::min(readed.front().size(), towrite)); if(cb.size() == 0) { return ret; @@ -718,16 +718,17 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte buffer = utils::page_allocator<byte>().allocate(blocksize); } deadline nd; - buffer_type b(buffer, (size_type) thisblock); + buffer_type b(buffer, utils::round_up_to_page_size(thisblock, 4096) /* to allow aligned i/o files */); LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); OUTCOME_TRY(auto &&readed, read({{&b, 1}, item.src.offset + thisoffset}, nd)); buffer_dirty = true; - if(readed.front().size() != thisblock) + if(readed.front().size() < thisblock) { return errc::resource_unavailable_try_again; // something is wrong } + readed.front() = {readed.front().data(), thisblock}; LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); - const_buffer_type cb(readed.front()); + const_buffer_type cb(readed.front().data(), thisblock); if(item.destination_extents_are_new) { // If we don't need to reset the bytes in the destination, try to elide diff --git a/include/llfio/v2.0/detail/impl/windows/fs_handle.ipp b/include/llfio/v2.0/detail/impl/windows/fs_handle.ipp index 9365a363..08628146 100644 --- a/include/llfio/v2.0/detail/impl/windows/fs_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/fs_handle.ipp @@ -83,10 +83,10 @@ result<path_handle> fs_handle::parent_path_handle(deadline d) const noexcept DWORD fileshare = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(filename, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(filename); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); OBJECT_ATTRIBUTES oa{}; memset(&oa, 0, sizeof(oa)); oa.Length = sizeof(OBJECT_ATTRIBUTES); @@ -139,9 +139,9 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo // If the target is a win32 path, we need to convert to NT path and call ourselves if(!base.is_valid() && !path.is_ntpath()) { - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); UNICODE_STRING NtPath{}; - if(RtlDosPathNameToNtPathName_U(zpath.buffer, &NtPath, nullptr, nullptr) == 0u) + if(RtlDosPathNameToNtPathName_U(zpath.data(), &NtPath, nullptr, nullptr) == 0u) { return win32_error(ERROR_FILE_NOT_FOUND); } @@ -188,11 +188,11 @@ result<void> fs_handle::relink(const path_handle &base, path_view_type path, boo duph = h.native_handle().h; } - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -227,9 +227,9 @@ result<void> fs_handle::link(const path_handle &base, path_view_type path, deadl // If the target is a win32 path, we need to convert to NT path and call ourselves if(!base.is_valid() && !path.is_ntpath()) { - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); UNICODE_STRING NtPath{}; - if(RtlDosPathNameToNtPathName_U(zpath.buffer, &NtPath, nullptr, nullptr) == 0u) + if(RtlDosPathNameToNtPathName_U(zpath.data(), &NtPath, nullptr, nullptr) == 0u) { return win32_error(ERROR_FILE_NOT_FOUND); } @@ -245,11 +245,11 @@ result<void> fs_handle::link(const path_handle &base, path_view_type path, deadl HANDLE duph = h.native_handle().h; - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -526,9 +526,9 @@ LLFIO_HEADERS_ONLY_FUNC_SPEC result<filesystem::path> to_win32_path(const fs_han #if(_HAS_CXX17 || __cplusplus >= 201700) && (!defined(__GLIBCXX__) || __GLIBCXX__ > 20170519) // libstdc++'s string_view is missing constexpr constexpr #endif - const wstring_view reserved_names[] = { - L"\\CON\\", L"\\PRN\\", L"\\AUX\\", L"\\NUL\\", L"\\COM1\\", L"\\COM2\\", L"\\COM3\\", L"\\COM4\\", L"\\COM5\\", L"\\COM6\\", L"\\COM7\\", - L"\\COM8\\", L"\\COM9\\", L"\\LPT1\\", L"\\LPT2\\", L"\\LPT3\\", L"\\LPT4\\", L"\\LPT5\\", L"\\LPT6\\", L"\\LPT7\\", L"\\LPT8\\", L"\\LPT9\\"}; + const wstring_view reserved_names[] = {L"\\CON\\", L"\\PRN\\", L"\\AUX\\", L"\\NUL\\", L"\\COM1\\", L"\\COM2\\", L"\\COM3\\", L"\\COM4\\", + L"\\COM5\\", L"\\COM6\\", L"\\COM7\\", L"\\COM8\\", L"\\COM9\\", L"\\LPT1\\", L"\\LPT2\\", L"\\LPT3\\", + L"\\LPT4\\", L"\\LPT5\\", L"\\LPT6\\", L"\\LPT7\\", L"\\LPT8\\", L"\\LPT9\\"}; wstring_view _buffer_(buffer); for(auto name : reserved_names) { diff --git a/include/llfio/v2.0/detail/impl/windows/import.hpp b/include/llfio/v2.0/detail/impl/windows/import.hpp index 71001ce6..dd4b0ac2 100644 --- a/include/llfio/v2.0/detail/impl/windows/import.hpp +++ b/include/llfio/v2.0/detail/impl/windows/import.hpp @@ -539,6 +539,26 @@ namespace windows_nt_kernel using DiscardVirtualMemory_t = BOOL(NTAPI *)(_In_ PVOID VirtualAddress, _In_ SIZE_T Size); + typedef struct _PERFORMANCE_INFORMATION + { + DWORD cb; + SIZE_T CommitTotal; + SIZE_T CommitLimit; + SIZE_T CommitPeak; + SIZE_T PhysicalTotal; + SIZE_T PhysicalAvailable; + SIZE_T SystemCache; + SIZE_T KernelTotal; + SIZE_T KernelPaged; + SIZE_T KernelNonpaged; + SIZE_T PageSize; + DWORD HandleCount; + DWORD ProcessCount; + DWORD ThreadCount; + } PERFORMANCE_INFORMATION, *PPERFORMANCE_INFORMATION, PERFORMACE_INFORMATION, *PPERFORMACE_INFORMATION; + + using GetPerformanceInfo_t = BOOL(NTAPI*)(PPERFORMANCE_INFORMATION pPerformanceInformation, DWORD cb); + using RtlCaptureStackBackTrace_t = USHORT(NTAPI *)(_In_ ULONG FramesToSkip, _In_ ULONG FramesToCapture, _Out_ PVOID *BackTrace, _Out_opt_ PULONG BackTraceHash); @@ -965,6 +985,7 @@ namespace windows_nt_kernel static AdjustTokenPrivileges_t AdjustTokenPrivileges; static PrefetchVirtualMemory_t PrefetchVirtualMemory_; static DiscardVirtualMemory_t DiscardVirtualMemory_; + static GetPerformanceInfo_t GetPerformanceInfo; static SymInitialize_t SymInitialize; static SymGetLineFromAddr64_t SymGetLineFromAddr64; static RtlCaptureStackBackTrace_t RtlCaptureStackBackTrace; @@ -1300,6 +1321,13 @@ namespace windows_nt_kernel { DiscardVirtualMemory_ = reinterpret_cast<DiscardVirtualMemory_t>(GetProcAddress(kernel32, "DiscardVirtualMemory")); } + if(GetPerformanceInfo == nullptr) + { + if((GetPerformanceInfo = reinterpret_cast<GetPerformanceInfo_t>(GetProcAddress(kernel32, "K32GetPerformanceInfo"))) == nullptr) + { + abort(); + } + } #ifdef LLFIO_OP_STACKBACKTRACEDEPTH if(dbghelp) { diff --git a/include/llfio/v2.0/detail/impl/windows/io_handle.ipp b/include/llfio/v2.0/detail/impl/windows/io_handle.ipp index 65c7237b..dba06f24 100644 --- a/include/llfio/v2.0/detail/impl/windows/io_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/io_handle.ipp @@ -112,7 +112,8 @@ inline bool do_read_write(io_handle::io_result<BuffersType> &ret, Syscall &&sysc if(nativeh.requires_aligned_io()) { assert(((uintptr_t) req.data() & 511) == 0); - assert((req.size() & 511) == 0); + // Reads must use aligned length as well + assert(std::is_const<typename BuffersType::value_type>::value || (req.size() & 511) == 0); } #endif reqs.offset += req.size(); diff --git a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp index 2ee3e5e3..b10539c3 100644 --- a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp @@ -682,15 +682,32 @@ result<map_handle> map_handle::map(section_handle §ion, size_type bytes, ext { windows_nt_kernel::init(); using namespace windows_nt_kernel; + OUTCOME_TRY(auto &&length, section.length()); // length of the backing file + if(length <= offset) + { + length = 0; + } + else + { + length -= offset; + } + if(bytes == 0u) + { + bytes = length; + } + else if(length > bytes) + { + length = bytes; + } result<map_handle> ret{map_handle(§ion, _flag)}; native_handle_type &nativeh = ret.value()._v; ULONG allocation = 0, prot; PVOID addr = nullptr; - size_t commitsize = bytes; + size_t commitsize = bytes + (offset & 65535); LARGE_INTEGER _offset{}; - _offset.QuadPart = offset; + _offset.QuadPart = offset & ~65535; OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag)); - SIZE_T _bytes = bytes; + SIZE_T _bytes = bytes + (offset & 65535); OUTCOME_TRY(win32_map_flags(nativeh, allocation, prot, commitsize, section.backing() != nullptr, ret.value()._flag)); LLFIO_LOG_FUNCTION_CALL(&ret); NTSTATUS ntstat = NtMapViewOfSection(section.native_handle().h, GetCurrentProcess(), &addr, 0, commitsize, &_offset, &_bytes, ViewUnmap, allocation, prot); @@ -700,8 +717,8 @@ result<map_handle> map_handle::map(section_handle §ion, size_type bytes, ext } ret.value()._addr = static_cast<byte *>(addr); ret.value()._offset = offset; - ret.value()._reservation = _bytes; - ret.value()._length = (size_type)(section.length().value() - offset); + ret.value()._reservation = _bytes - (offset & 65535); + ret.value()._length = length; ret.value()._pagesize = pagesize; // Make my handle borrow the native handle of my backing storage ret.value()._v.h = section.backing_native_handle().h; @@ -766,12 +783,24 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool /* un // So this must be file backed memory. Totally different APIs for that :) OUTCOME_TRY(auto &&length, _section->length()); // length of the backing file + if(length <= _offset) + { + length = 0; + } + else + { + length -= _offset; + } + if(length > _reservation) + { + length = _reservation; + } if(newsize < _reservation) { // If newsize isn't exactly a previous extension, this will fail, same as for the VirtualAlloc case OUTCOME_TRY(win32_release_file_allocations(_addr + newsize, _reservation - newsize)); _reservation = newsize; - _length = (size_type)((length - _offset < newsize) ? (length - _offset) : newsize); // length of backing, not reservation + _length = length; return _reservation; } // Try to map an additional part of the section directly after this map @@ -779,7 +808,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool /* un PVOID addr = _addr + _reservation; size_t commitsize = newsize - _reservation; LARGE_INTEGER offset{}; - offset.QuadPart = _offset + _reservation; + offset.QuadPart = (_offset + _reservation) & ~65535; SIZE_T _bytes = newsize - _reservation; native_handle_type nativeh; OUTCOME_TRY(win32_map_flags(nativeh, allocation, prot, commitsize, _section->backing() != nullptr, _flag)); @@ -789,7 +818,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool /* un return ntkernel_error(ntstat); } _reservation += _bytes; - _length = (size_type)((length - _offset < newsize) ? (length - _offset) : newsize); // length of backing, not reservation + _length = length; return _reservation; } @@ -963,7 +992,7 @@ result<map_handle::buffer_type> map_handle::do_not_store(buffer_type region) noe map_handle::io_result<map_handle::buffers_type> map_handle::_do_read(io_request<buffers_type> reqs, deadline /*d*/) noexcept { LLFIO_LOG_FUNCTION_CALL(this); - byte *addr = _addr + reqs.offset; + byte *addr = _addr + reqs.offset + (_offset & 65535); size_type togo = reqs.offset < _length ? static_cast<size_type>(_length - reqs.offset) : 0; for(size_t i = 0; i < reqs.buffers.size(); i++) { @@ -986,21 +1015,23 @@ map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_write(io_r LLFIO_LOG_FUNCTION_CALL(this); if(!!(_flag & section_handle::flag::write_via_syscall) && _section != nullptr && _section->backing() != nullptr) { + reqs.offset += _offset; auto r = _section->backing()->write(reqs, d); if(!r) { return std::move(r).error(); } + reqs.offset -= _offset; if(reqs.offset + r.bytes_transferred() > _length) { OUTCOME_TRY(update_map()); } return std::move(r).value(); } - byte *addr = _addr + reqs.offset; + byte *addr = _addr + reqs.offset + (_offset & 65535); size_type togo = reqs.offset < _length ? static_cast<size_type>(_length - reqs.offset) : 0; if(QUICKCPPLIB_NAMESPACE::signal_guard::signal_guard( - QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::undefined_memory_access, + QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::undefined_memory_access | QUICKCPPLIB_NAMESPACE::signal_guard::signalc_set::segmentation_fault, [&] { for(size_t i = 0; i < reqs.buffers.size(); i++) { diff --git a/include/llfio/v2.0/detail/impl/windows/mapped_file_handle.ipp b/include/llfio/v2.0/detail/impl/windows/mapped_file_handle.ipp index 0bdaf4a5..c78cdc05 100644 --- a/include/llfio/v2.0/detail/impl/windows/mapped_file_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/mapped_file_handle.ipp @@ -74,7 +74,7 @@ result<mapped_file_handle::size_type> mapped_file_handle::_reserve(extent_type & map_size = (size_type) length; } OUTCOME_TRYV(_mh.close()); - OUTCOME_TRY(auto &&mh, map_handle::map(_sh, map_size, 0, mapflags)); + OUTCOME_TRY(auto &&mh, map_handle::map(_sh, map_size, _offset, mapflags)); _mh = std::move(mh); _reservation = reservation; return _reservation; diff --git a/include/llfio/v2.0/detail/impl/windows/path_handle.ipp b/include/llfio/v2.0/detail/impl/windows/path_handle.ipp index 2d394c8f..44548b6a 100644 --- a/include/llfio/v2.0/detail/impl/windows/path_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/path_handle.ipp @@ -34,11 +34,11 @@ result<bool> path_handle::exists(path_view_type path) const noexcept windows_nt_kernel::init(); using namespace windows_nt_kernel; LLFIO_LOG_FUNCTION_CALL(this); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -99,11 +99,11 @@ result<path_handle> path_handle::path(const path_handle &base, path_handle::path ntflags |= 0x01 /*FILE_DIRECTORY_FILE*/; // required to open a directory IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -135,8 +135,8 @@ result<path_handle> path_handle::path(const path_handle &base, path_handle::path { DWORD creation = OPEN_EXISTING; attribs |= FILE_FLAG_BACKUP_SEMANTICS; // required to open a directory - path_view::c_str<> zpath(path, path_view::zero_terminated); - if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.buffer, access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT + path_view::zero_terminated_rendered_path<> zpath(path); + if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.data(), access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT { DWORD errcode = GetLastError(); // assert(false); diff --git a/include/llfio/v2.0/detail/impl/windows/pipe_handle.ipp b/include/llfio/v2.0/detail/impl/windows/pipe_handle.ipp index 6f8c5d1f..986618b4 100644 --- a/include/llfio/v2.0/detail/impl/windows/pipe_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/pipe_handle.ipp @@ -72,7 +72,7 @@ result<pipe_handle> pipe_handle::pipe(pipe_handle::path_view_type path, pipe_han ntflags &= ~0x00000008 /*FILE_NO_INTERMEDIATE_BUFFERING*/; // pipes always buffer IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; if(path.empty()) { @@ -80,9 +80,9 @@ result<pipe_handle> pipe_handle::pipe(pipe_handle::path_view_type path, pipe_han } else { - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); diff --git a/include/llfio/v2.0/detail/impl/windows/process_handle.ipp b/include/llfio/v2.0/detail/impl/windows/process_handle.ipp index ebc63d00..a6679209 100644 --- a/include/llfio/v2.0/detail/impl/windows/process_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/process_handle.ipp @@ -268,9 +268,9 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<process_handle> process_handle::launch_pr })); } *envbuffere = 0; - path_view::c_str<> zpath(path, path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(path); PROCESS_INFORMATION pi; - if(!CreateProcessW(zpath.buffer, argsbuffer, nullptr, nullptr, true, CREATE_UNICODE_ENVIRONMENT, envbuffer, nullptr, &si, &pi)) + if(!CreateProcessW(zpath.data(), argsbuffer, nullptr, nullptr, true, CREATE_UNICODE_ENVIRONMENT, envbuffer, nullptr, &si, &pi)) return win32_error(); nativeh.h = pi.hProcess; (void) CloseHandle(pi.hThread); diff --git a/include/llfio/v2.0/detail/impl/windows/symlink_handle.ipp b/include/llfio/v2.0/detail/impl/windows/symlink_handle.ipp index 97dde7b3..e239d2fd 100644 --- a/include/llfio/v2.0/detail/impl/windows/symlink_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/symlink_handle.ipp @@ -108,11 +108,11 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<symlink_handle> symlink_handle::symlink(c ntflags |= 0x4000 /*FILE_OPEN_FOR_BACKUP_INTENT*/ | 0x00200000 /*FILE_OPEN_REPARSE_POINT*/; IO_STATUS_BLOCK isb = make_iostatus(); - path_view::c_str<> zpath(path, path_view::not_zero_terminated); + path_view::not_zero_terminated_rendered_path<> zpath(path); UNICODE_STRING _path{}; - _path.Buffer = const_cast<wchar_t *>(zpath.buffer); - _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.length * sizeof(wchar_t))) + sizeof(wchar_t); - if(zpath.length >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') + _path.Buffer = const_cast<wchar_t *>(zpath.data()); + _path.MaximumLength = (_path.Length = static_cast<USHORT>(zpath.size() * sizeof(wchar_t))) + sizeof(wchar_t); + if(zpath.size() >= 4 && _path.Buffer[0] == '\\' && _path.Buffer[1] == '!' && _path.Buffer[2] == '!' && _path.Buffer[3] == '\\') { _path.Buffer += 3; _path.Length -= 3 * sizeof(wchar_t); @@ -162,8 +162,8 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<symlink_handle> symlink_handle::symlink(c } // required to open a symlink attribs |= FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT; - path_view::c_str<> zpath(path, path_view::zero_terminated); - if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.buffer, access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT + path_view::zero_terminated_rendered_path<> zpath(path); + if(INVALID_HANDLE_VALUE == (nativeh.h = CreateFileW_(zpath.data(), access, fileshare, nullptr, creation, attribs, nullptr))) // NOLINT { DWORD errcode = GetLastError(); // assert(false); @@ -254,7 +254,7 @@ result<symlink_handle::const_buffers_type> symlink_handle::write(symlink_handle: auto *buffer = req.kernelbuffer.empty() ? alloca(buffersize) : req.kernelbuffer.data(); memset(buffer, 0, sizeof(REPARSE_DATA_BUFFER)); auto *rpd = (REPARSE_DATA_BUFFER *) buffer; - path_view::c_str<> zpath(req.buffers.path(), path_view::zero_terminated); + path_view::zero_terminated_rendered_path<> zpath(req.buffers.path()); switch(req.buffers.type()) { case symlink_type::none: @@ -263,24 +263,24 @@ result<symlink_handle::const_buffers_type> symlink_handle::write(symlink_handle: { const size_t reparsebufferheaderlen = offsetof(REPARSE_DATA_BUFFER, SymbolicLinkReparseBuffer.PathBuffer) - headerlen; rpd->ReparseTag = IO_REPARSE_TAG_SYMLINK; - if(zpath.length >= 4 && zpath.buffer[0] == '\\' && zpath.buffer[1] == '!' && zpath.buffer[2] == '!' && zpath.buffer[3] == '\\') + if(zpath.size() >= 4 && zpath.data()[0] == '\\' && zpath.data()[1] == '!' && zpath.data()[2] == '!' && zpath.data()[3] == '\\') { - memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer, zpath.buffer + 3, destpathbytes - 6 + sizeof(wchar_t)); + memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer, zpath.data() + 3, destpathbytes - 6 + sizeof(wchar_t)); rpd->SymbolicLinkReparseBuffer.SubstituteNameOffset = 0; rpd->SymbolicLinkReparseBuffer.SubstituteNameLength = (USHORT) destpathbytes - 6; rpd->SymbolicLinkReparseBuffer.PrintNameOffset = (USHORT)(destpathbytes - 6 + sizeof(wchar_t)); rpd->SymbolicLinkReparseBuffer.PrintNameLength = (USHORT) destpathbytes - 6; - memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer + rpd->SymbolicLinkReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.buffer + 3, + memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer + rpd->SymbolicLinkReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.data() + 3, rpd->SymbolicLinkReparseBuffer.PrintNameLength - 6 + sizeof(wchar_t)); } else { - memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer, zpath.buffer, destpathbytes + sizeof(wchar_t)); + memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer, zpath.data(), destpathbytes + sizeof(wchar_t)); rpd->SymbolicLinkReparseBuffer.SubstituteNameOffset = 0; rpd->SymbolicLinkReparseBuffer.SubstituteNameLength = (USHORT) destpathbytes; rpd->SymbolicLinkReparseBuffer.PrintNameOffset = (USHORT)(destpathbytes + sizeof(wchar_t)); rpd->SymbolicLinkReparseBuffer.PrintNameLength = (USHORT) destpathbytes; - memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer + rpd->SymbolicLinkReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.buffer, + memcpy(rpd->SymbolicLinkReparseBuffer.PathBuffer + rpd->SymbolicLinkReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.data(), rpd->SymbolicLinkReparseBuffer.PrintNameLength + sizeof(wchar_t)); } rpd->SymbolicLinkReparseBuffer.Flags = req.buffers.path().is_relative() ? 0x1 /*SYMLINK_FLAG_RELATIVE*/ : 0; @@ -295,24 +295,24 @@ result<symlink_handle::const_buffers_type> symlink_handle::write(symlink_handle: { const size_t reparsebufferheaderlen = offsetof(REPARSE_DATA_BUFFER, MountPointReparseBuffer.PathBuffer) - headerlen; rpd->ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; - if(zpath.length >= 4 && zpath.buffer[0] == '\\' && zpath.buffer[1] == '!' && zpath.buffer[2] == '!' && zpath.buffer[3] == '\\') + if(zpath.size() >= 4 && zpath.data()[0] == '\\' && zpath.data()[1] == '!' && zpath.data()[2] == '!' && zpath.data()[3] == '\\') { - memcpy(rpd->MountPointReparseBuffer.PathBuffer, zpath.buffer + 3, destpathbytes - 6 + sizeof(wchar_t)); + memcpy(rpd->MountPointReparseBuffer.PathBuffer, zpath.data() + 3, destpathbytes - 6 + sizeof(wchar_t)); rpd->MountPointReparseBuffer.SubstituteNameOffset = 0; rpd->MountPointReparseBuffer.SubstituteNameLength = (USHORT) destpathbytes - 6; rpd->MountPointReparseBuffer.PrintNameOffset = (USHORT)(destpathbytes - 6 + sizeof(wchar_t)); rpd->MountPointReparseBuffer.PrintNameLength = (USHORT) destpathbytes - 6; - memcpy(rpd->MountPointReparseBuffer.PathBuffer + rpd->MountPointReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.buffer + 3, + memcpy(rpd->MountPointReparseBuffer.PathBuffer + rpd->MountPointReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.data() + 3, rpd->MountPointReparseBuffer.PrintNameLength - 6 + sizeof(wchar_t)); } else { - memcpy(rpd->MountPointReparseBuffer.PathBuffer, zpath.buffer, destpathbytes + sizeof(wchar_t)); + memcpy(rpd->MountPointReparseBuffer.PathBuffer, zpath.data(), destpathbytes + sizeof(wchar_t)); rpd->MountPointReparseBuffer.SubstituteNameOffset = 0; rpd->MountPointReparseBuffer.SubstituteNameLength = (USHORT) destpathbytes; rpd->MountPointReparseBuffer.PrintNameOffset = (USHORT)(destpathbytes + sizeof(wchar_t)); rpd->MountPointReparseBuffer.PrintNameLength = (USHORT) destpathbytes; - memcpy(rpd->MountPointReparseBuffer.PathBuffer + rpd->MountPointReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.buffer, + memcpy(rpd->MountPointReparseBuffer.PathBuffer + rpd->MountPointReparseBuffer.PrintNameOffset / sizeof(wchar_t), zpath.data(), rpd->MountPointReparseBuffer.PrintNameLength + sizeof(wchar_t)); } rpd->ReparseDataLength = diff --git a/include/llfio/v2.0/detail/impl/windows/utils.ipp b/include/llfio/v2.0/detail/impl/windows/utils.ipp index e12db462..4557e567 100644 --- a/include/llfio/v2.0/detail/impl/windows/utils.ipp +++ b/include/llfio/v2.0/detail/impl/windows/utils.ipp @@ -207,38 +207,55 @@ namespace utils return success(); } - result<process_memory_usage> current_process_memory_usage(process_memory_usage::want /*unused*/) noexcept + result<process_memory_usage> current_process_memory_usage(process_memory_usage::want wanted) noexcept { // Amazingly Win32 doesn't expose private working set, so to avoid having // to iterate all the pages in the process and calculate, use a hidden // NT kernel call windows_nt_kernel::init(); using namespace windows_nt_kernel; - ULONG written = 0; - _VM_COUNTERS_EX2 vmc; - memset(&vmc, 0, sizeof(vmc)); - NTSTATUS ntstat = NtQueryInformationProcess(GetCurrentProcess(), ProcessVmCounters, &vmc, sizeof(vmc), &written); - if(ntstat < 0) - { - return ntkernel_error(ntstat); - } process_memory_usage ret; - /* Notes: + if(!!(wanted & process_memory_usage::want::this_process)) + { + ULONG written = 0; + _VM_COUNTERS_EX2 vmc; + memset(&vmc, 0, sizeof(vmc)); + NTSTATUS ntstat = NtQueryInformationProcess(GetCurrentProcess(), ProcessVmCounters, &vmc, sizeof(vmc), &written); + if(ntstat < 0) + { + return ntkernel_error(ntstat); + } + /* Notes: - Apparently PrivateUsage is the commit charge on Windows. It always equals PagefileUsage. - It is the total amount of private anonymous pages committed. - SharedCommitUsage is amount of non-binary Shared memory committed. - Therefore total non-binary commit = PrivateUsage + SharedCommitUsage + Apparently PrivateUsage is the commit charge on Windows. It always equals PagefileUsage. + It is the total amount of private anonymous pages committed. + SharedCommitUsage is amount of non-binary Shared memory committed. + Therefore total non-binary commit = PrivateUsage + SharedCommitUsage - WorkingSetSize is the total amount of program binaries, non-binary shared memory, and anonymous pages faulted in. - PrivateWorkingSetSize is the amount of private anonymous pages faulted into the process. - Therefore remainder is all shared working set faulted into the process. - */ - ret.total_address_space_in_use = vmc.VirtualSize; - ret.total_address_space_paged_in = vmc.WorkingSetSize; + WorkingSetSize is the total amount of program binaries, non-binary shared memory, and anonymous pages faulted in. + PrivateWorkingSetSize is the amount of private anonymous pages faulted into the process. + Therefore remainder is all shared working set faulted into the process. + */ + ret.total_address_space_in_use = vmc.VirtualSize; + ret.total_address_space_paged_in = vmc.WorkingSetSize; - ret.private_committed = vmc.PrivateUsage; - ret.private_paged_in = vmc.PrivateWorkingSetSize; + ret.private_committed = vmc.PrivateUsage; + ret.private_paged_in = vmc.PrivateWorkingSetSize; + } + if(!!(wanted & process_memory_usage::want::this_system)) + { + PERFORMANCE_INFORMATION pi; + memset(&pi, 0, sizeof(pi)); + pi.cb = sizeof(pi); + if(!GetPerformanceInfo(&pi, sizeof(pi))) + { + return win32_error(); + } + ret.system_physical_memory_total = (uint64_t) pi.PhysicalTotal * pi.PageSize; + ret.system_physical_memory_available = (uint64_t) pi.PhysicalAvailable * pi.PageSize; + ret.system_commit_charge_maximum = (uint64_t) pi.CommitLimit * pi.PageSize; + ret.system_commit_charge_available = (uint64_t)(pi.CommitLimit - pi.CommitTotal) * pi.PageSize; + } return ret; } diff --git a/include/llfio/v2.0/fs_handle.hpp b/include/llfio/v2.0/fs_handle.hpp index 1be9bdaf..f6a6630b 100644 --- a/include/llfio/v2.0/fs_handle.hpp +++ b/include/llfio/v2.0/fs_handle.hpp @@ -120,8 +120,29 @@ change in the future, however any path emitted will always be a valid Win32 path */ #ifdef _WIN32 LLFIO_HEADERS_ONLY_FUNC_SPEC result<filesystem::path> to_win32_path(const fs_handle &h, win32_path_namespace mapping = win32_path_namespace::any) noexcept; +LLFIO_TEMPLATE(class T) +LLFIO_TREQUIRES(LLFIO_TPRED(!std::is_base_of<fs_handle, T>::value && std::is_base_of<handle, T>::value)) +inline result<filesystem::path> to_win32_path(const T &h, win32_path_namespace mapping = win32_path_namespace::any) noexcept +{ + struct wrapper final : public fs_handle + { + const handle &_h; + explicit wrapper(const handle &h) + : _h(h) + { + } + virtual const handle &_get_handle() const noexcept override { return _h; } + } _(h); + return to_win32_path(_, mapping); +} #else inline result<filesystem::path> to_win32_path(const fs_handle &h, win32_path_namespace mapping = win32_path_namespace::any) noexcept; +LLFIO_TEMPLATE(class T) +LLFIO_TREQUIRES(LLFIO_TPRED(!std::is_base_of<fs_handle, T>::value && std::is_base_of<handle, T>::value)) +inline result<filesystem::path> to_win32_path(const T &h, win32_path_namespace /*unused*/ = win32_path_namespace::any) noexcept +{ + return h.current_path(); +} #endif /*! \class fs_handle diff --git a/include/llfio/v2.0/map_handle.hpp b/include/llfio/v2.0/map_handle.hpp index 01bee2ff..55cefa8f 100644 --- a/include/llfio/v2.0/map_handle.hpp +++ b/include/llfio/v2.0/map_handle.hpp @@ -660,9 +660,8 @@ public: \param section A memory section handle specifying the backing storage to use. \param bytes How many bytes to reserve (0 = the size of the section). Rounded up to nearest 64Kb on Windows. - \param offset The offset into the backing storage to map from. Typically needs to be at least - a multiple of the page size (see `page_size()`), on Windows it needs to be a multiple of the - kernel memory allocation granularity (typically 64Kb). + \param offset The offset into the backing storage to map from. This can be byte granularity, + but be careful if you use non-pagesize offsets (see below). \param _flag The permissions with which to map the view which are constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use `commit()` to later change availability of memory. @@ -708,6 +707,10 @@ public: */ static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than = {}, size_t max_items = (size_t) -1) noexcept; + /*! Disable the map handle cache, returning its previous setting. Note that you may also + wish to explicitly trim the cache. + */ + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC bool set_cache_disabled(bool disabled) noexcept; //! The memory section this handle is using section_handle *section() const noexcept { return _section; } @@ -715,7 +718,16 @@ public: void set_section(section_handle *s) noexcept { _section = s; } //! The address in memory where this mapped view resides - byte *address() const noexcept { return _addr; } + byte *address() const noexcept + { +#ifdef _WIN32 + if(_pagesize <= 65536) + { + return _addr + (_offset & 65535); + } +#endif + return _addr + (_offset & (_pagesize - 1)); + } //! The offset of the memory map. extent_type offset() const noexcept { return _offset; } @@ -746,7 +758,14 @@ public: return _reservation; } OUTCOME_TRY(auto &&length, _section->length()); // length of the backing file - length -= _offset; + if(length <= _offset) + { + length = 0; + } + else + { + length -= _offset; + } if(length > _reservation) { length = _reservation; @@ -1094,8 +1113,8 @@ inline result<map_handle> map(map_handle::size_type bytes, bool zeroed = false, /*! Create a memory mapped view of a backing storage, optionally reserving additional address space for later growth. \param section A memory section handle specifying the backing storage to use. \param bytes How many bytes to reserve (0 = the size of the section). Rounded up to nearest 64Kb on Windows. -\param offset The offset into the backing storage to map from. Typically needs to be at least a multiple of the page size (see `page_size()`), on Windows it -needs to be a multiple of the kernel memory allocation granularity (typically 64Kb). \param _flag The permissions with which to map the view which are +\param offset The offset into the backing storage to map from +\param _flag The permissions with which to map the view which are constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use commit() to later change availability of memory. diff --git a/include/llfio/v2.0/mapped_file_handle.hpp b/include/llfio/v2.0/mapped_file_handle.hpp index ce92acab..423ef906 100644 --- a/include/llfio/v2.0/mapped_file_handle.hpp +++ b/include/llfio/v2.0/mapped_file_handle.hpp @@ -55,8 +55,8 @@ than memory maps. However for lots of say 64 byte i/o, the gain of memory maps o syscalls is unsurpassable. This class combines a `file_handle` with a `section_handle` and a `map_handle` to -implement a fully memory mapped `file_handle`. The whole file is always mapped entirely -into memory, and `read()` and `write()` i/o is performed directly with the map. +implement a fully memory mapped `file_handle`. The portion of the file between `starting_offset()` +and `capacity()` is mapped into memory, and `read()` and `write()` i/o is performed directly with the map. Reads always return the original mapped data, and do not fill any buffers passed in. For obvious reasons the utility of this class on 32-bit systems is limited, but can be useful when used with smaller files. @@ -104,6 +104,18 @@ length, the map is updated to match the underlying file, up to the reservation l You can of course explicitly call `update_map()` whenever you need the map to reflect changes to the maximum extent of the underlying file. +`starting_offset()` offsets all map i/o, truncation, and sizing, it is always as if the +underlying file starts from that offset. This *only* applies to the `map_handle` portion, +the `section_handle` portion has no concept of starting offsets. Starting offsets can +be byte granularity, the underlying map will use an appropriate platform specific +granularity (typically the page size on POSIX and 64Kb on Windows). You might note that +combined with how reservations work, this allows a `mapped_file_handle` to wholly reflect +a subset of a very large file -- only the relevant portion of that very large file will +be mapped into memory, `read()` and `write()` will not exceed the boundaries of +`starting_offset() - capacity()`, and `maximum_extent()` will not return a value +exceeding the reservation. This effectively means such a subset file quacks like a +`file_handle` of a complete file reflecting just that subset proportion. + It is up to you to detect that the reservation has been exhausted, and to reserve a new reservation which will change the value returned by `address()`. This entirely manual system is a bit tedious and cumbersome to use, but as mapping files @@ -174,6 +186,7 @@ public: protected: size_type _reservation{0}; + extent_type _offset{0}; section_handle _sh; // Tracks the file (i.e. *this) somewhat lazily map_handle _mh; // The current map with valid extent @@ -227,7 +240,8 @@ protected: assert(_mh.native_handle()._init == native_handle()._init); if(!!(_sh.section_flags() & section_handle::flag::write_via_syscall)) { - const auto batch = max_buffers(); + const auto batch = file_handle::_do_max_buffers(); + reqs.offset += _offset; io_request<const_buffers_type> thisreq(reqs); LLFIO_DEADLINE_TO_SLEEP_INIT(d); for(size_t n = 0; n < reqs.buffers.size();) @@ -235,6 +249,11 @@ protected: deadline nd; LLFIO_DEADLINE_TO_PARTIAL_DEADLINE(nd, d); thisreq.buffers = reqs.buffers.subspan(n, std::min(batch, reqs.buffers.size() - n)); + if(thisreq.buffers.size() == 1 && thisreq.buffers.front().size() == 0) + { + n++; + continue; + } OUTCOME_TRY(auto &&written, file_handle::_do_write(thisreq, nd)); if(written.empty()) { @@ -266,6 +285,7 @@ public: mapped_file_handle(mapped_file_handle &&o) noexcept : file_handle(std::move(o)) , _reservation(o._reservation) + , _offset(o._offset) , _sh(std::move(o._sh)) , _mh(std::move(o._mh)) { @@ -281,14 +301,22 @@ public: //! No copy construction (use `clone()`) mapped_file_handle(const mapped_file_handle &) = delete; //! Explicit conversion from file_handle permitted - explicit constexpr mapped_file_handle(file_handle &&o, section_handle::flag sflags) noexcept + LLFIO_TEMPLATE(class SHF) + LLFIO_TREQUIRES(LLFIO_TPRED(std::is_same<typename std::decay<SHF>::type, section_handle::flag>::value || + std::is_same<typename std::decay<SHF>::type, section_handle::flag::enum_type>::value)) + explicit constexpr mapped_file_handle(file_handle &&o, SHF sflags, extent_type offset) noexcept : file_handle(std::move(o)) + , _offset(offset) , _sh(sflags) { } //! Explicit conversion from file_handle permitted, this overload also attempts to map the file - explicit mapped_file_handle(file_handle &&o, size_type reservation, section_handle::flag sflags) + LLFIO_TEMPLATE(class SHF) + LLFIO_TREQUIRES(LLFIO_TPRED(std::is_same<typename std::decay<SHF>::type, section_handle::flag>::value || + std::is_same<typename std::decay<SHF>::type, section_handle::flag::enum_type>::value)) + explicit mapped_file_handle(file_handle &&o, size_type reservation, SHF sflags, extent_type offset) : file_handle(std::move(o)) + , _offset(offset) , _sh(sflags) { auto length = (extent_type) -1; @@ -355,7 +383,7 @@ public: LLFIO_MAKE_FREE_FUNCTION static inline result<mapped_file_handle> mapped_file(size_type reservation, const path_handle &base, path_view_type _path, mode _mode = mode::read, creation _creation = creation::open_existing, caching _caching = caching::all, flag flags = flag::none, - section_handle::flag sflags = section_handle::flag::none) noexcept + section_handle::flag sflags = section_handle::flag::none, extent_type offset = 0) noexcept { try { @@ -369,7 +397,7 @@ public: default: { // Attempt mapping now (may silently fail if file is empty) - mapped_file_handle mfh(std::move(fh), reservation, sflags); + mapped_file_handle mfh(std::move(fh), reservation, sflags, offset); return {std::move(mfh)}; } case creation::only_if_not_exist: @@ -377,7 +405,7 @@ public: case creation::always_new: { // Don't attempt mapping now as file will be empty - mapped_file_handle mfh(std::move(fh), sflags); + mapped_file_handle mfh(std::move(fh), sflags, offset); mfh._reservation = reservation; return {std::move(mfh)}; } @@ -392,9 +420,9 @@ public: LLFIO_MAKE_FREE_FUNCTION static inline result<mapped_file_handle> mapped_file(const path_handle &base, path_view_type _path, mode _mode = mode::read, creation _creation = creation::open_existing, caching _caching = caching::all, flag flags = flag::none, - section_handle::flag sflags = section_handle::flag::none) noexcept + section_handle::flag sflags = section_handle::flag::none, extent_type offset = 0) noexcept { - return mapped_file(0, base, _path, _mode, _creation, _caching, flags, sflags); + return mapped_file(0, base, _path, _mode, _creation, _caching, flags, sflags, offset); } /*! Create an mapped file handle creating a uniquely named file on a path. @@ -471,7 +499,7 @@ public: try { OUTCOME_TRY(auto &&v, file_handle::temp_inode(dir, _mode, flags)); - mapped_file_handle ret(std::move(v), sflags); + mapped_file_handle ret(std::move(v), sflags, 0); ret._reservation = reservation; return {std::move(ret)}; } @@ -491,8 +519,10 @@ public: //! The map this handle is using map_handle &map() noexcept { return _mh; } - //! The address in memory where this mapped file resides + //! The address in memory where this mapped file currently resides byte *address() const noexcept { return _mh.address(); } + //! The offset into the backing file from which this mapped file begins + extent_type starting_offset() const noexcept { return _offset; } //! The page size used by the map, in bytes. size_type page_size() const noexcept { return _mh.page_size(); } @@ -500,8 +530,16 @@ public: //! True if the map is of non-volatile RAM bool is_nvram() const noexcept { return _mh.is_nvram(); } - //! The maximum extent of the underlying file - result<extent_type> underlying_file_maximum_extent() const noexcept { return file_handle::maximum_extent(); } + //! The maximum extent of the underlying file, minus any offset. + result<extent_type> underlying_file_maximum_extent() const noexcept + { + OUTCOME_TRY(auto &&ret, file_handle::maximum_extent()); + if(ret <= _offset) + { + return 0; + } + return ret - _offset; + } //! The address space (to be) reserved for future expansion of this file. size_type capacity() const noexcept { return _reservation; } @@ -528,13 +566,13 @@ public: } LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> close() noexcept override; LLFIO_HEADERS_ONLY_VIRTUAL_SPEC native_handle_type release() noexcept override; - result<mapped_file_handle> reopen(size_type reservation, mode mode_ = mode::unchanged, caching caching_ = caching::unchanged, + result<mapped_file_handle> reopen(size_type reservation, extent_type offset = 0, mode mode_ = mode::unchanged, caching caching_ = caching::unchanged, deadline d = std::chrono::seconds(30)) const noexcept { try { OUTCOME_TRY(auto &&fh, file_handle::reopen(mode_, caching_, d)); - return mapped_file_handle(std::move(fh), reservation, _sh.section_flags()); + return mapped_file_handle(std::move(fh), reservation, _sh.section_flags(), _offset + offset); } catch(...) { @@ -651,6 +689,17 @@ public: LLFIO_DEADLINE_TRY_FOR_UNTIL(relink) }; +static_assert(!std::is_constructible<mapped_file_handle, file_handle, int, int>::value, "mapped_file_handle(file_handle, int, int) must not be constructible!"); +static_assert(!std::is_constructible<mapped_file_handle, file_handle, int, int, int>::value, + "mapped_file_handle(file_handle, int, int, int) must not be constructible!"); +static_assert(std::is_constructible<mapped_file_handle, file_handle, section_handle::flag, int>::value, + "mapped_file_handle(file_handle, section_handle::flag, int) must be constructible!"); +static_assert(std::is_constructible<mapped_file_handle, file_handle, section_handle::flag::enum_type, int>::value, + "mapped_file_handle(file_handle, section_handle::flag::enum_type, int) must be constructible!"); +static_assert(std::is_constructible<mapped_file_handle, file_handle, int, section_handle::flag, int>::value, + "mapped_file_handle(file_handle, int, section_handle::flag, int) must be constructible!"); +static_assert(std::is_constructible<mapped_file_handle, file_handle, int, section_handle::flag::enum_type, int>::value, + "mapped_file_handle(file_handle, int, section_handle::flag::enum_type, int) must be constructible!"); //! \brief Constructor for `mapped_file_handle` template <> struct construct<mapped_file_handle> diff --git a/include/llfio/v2.0/native_handle_type.hpp b/include/llfio/v2.0/native_handle_type.hpp index ccc9df4b..cd2a42cd 100644 --- a/include/llfio/v2.0/native_handle_type.hpp +++ b/include/llfio/v2.0/native_handle_type.hpp @@ -88,9 +88,12 @@ struct native_handle_type // NOLINT //! A third party pointer void *ptr; }; - disposition behaviour; //! The behaviour of the handle + disposition behaviour{disposition::invalid}; //! The behaviour of the handle //! Constructs a default instance - constexpr native_handle_type() {} // NOLINT + constexpr native_handle_type() + : _init{-1} + { + } // NOLINT ~native_handle_type() = default; //! Construct from a POSIX file descriptor constexpr native_handle_type(disposition _behaviour, int _fd) noexcept diff --git a/include/llfio/v2.0/path_view.hpp b/include/llfio/v2.0/path_view.hpp index 48618e3d..f5ae2044 100644 --- a/include/llfio/v2.0/path_view.hpp +++ b/include/llfio/v2.0/path_view.hpp @@ -1,5 +1,5 @@ /* A view of a path to something -(C) 2017-2020 Niall Douglas <http://www.nedproductions.biz/> (20 commits) +(C) 2017-2021 Niall Douglas <http://www.nedproductions.biz/> (20 commits) File Created: Jul 2017 @@ -215,7 +215,7 @@ public: //! i.e. `is_source_chartype_acceptable`, or is `byte` template <class Char> static constexpr bool is_source_acceptable = detail::is_source_acceptable<Char>::value; - //! The default internal buffer size used by `c_str`. + //! The default internal buffer size used by `rendered_path`. static constexpr size_t default_internal_buffer_size = 1024; // 2Kb for wchar_t, 1Kb for char //! How to interpret separators @@ -236,7 +236,7 @@ public: }; //! The default deleter to use - template <class T> using default_c_str_deleter = std::default_delete<T>; + template <class T> using default_rendered_path_deleter = std::default_delete<T>; private: static constexpr auto _npos = string_view::npos; @@ -718,24 +718,24 @@ private: { return a.compare(b); } - // Disparate source encodings compare via c_str + // Disparate source encodings compare via rendered_path template <class DestT, class Deleter, size_t _internal_buffer_size, class Char1T, class Char2T> static int _compare(basic_string_view<Char1T> a, enum zero_termination a_zt, basic_string_view<Char2T> b, enum zero_termination b_zt, const std::locale *loc) noexcept { path_view_component _a_(a.data(), a.size(), a_zt); path_view_component _b_(b.data(), b.size(), b_zt); - c_str<DestT, Deleter, _internal_buffer_size> _a(_a_, not_zero_terminated, loc); - c_str<DestT, Deleter, _internal_buffer_size> _b(_b_, not_zero_terminated, loc); - if(_a.length < _b.length) + rendered_path<zero_termination::not_zero_terminated, DestT, Deleter, _internal_buffer_size> _a(_a_, loc); + rendered_path<zero_termination::not_zero_terminated, DestT, Deleter, _internal_buffer_size> _b(_b_, loc); + if(_a.size() < _b.size()) { return -1; } - if(_a.length > _b.length) + if(_a.size() > _b.size()) { return 1; } - return _do_compare(_a.buffer, _b.buffer, _a.length); + return _do_compare(_a.data(), _b.data(), _a.size()); } public: @@ -772,17 +772,17 @@ public: If the source encodings of the two path views are compatible, a lexicographical comparison is performed. If they are incompatible, either or both views are converted to the destination encoding - using `c_str<T, Delete, _internal_buffer_size>`, and then a + using `rendered_path<T, Delete, _internal_buffer_size>`, and then a lexicographical comparison is performed. This can, for obvious reasons, be expensive. It can also throw - exceptions, as `c_str` does. + exceptions, as `rendered_path` does. If the destination encoding is `byte`, `memcmp()` is used, - and `c_str` is never invoked as the two sources are byte + and `rendered_path` is never invoked as the two sources are byte compared directly. */ - LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_c_str_deleter<T[]>, + LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_rendered_path_deleter<T[]>, size_t _internal_buffer_size = default_internal_buffer_size) LLFIO_TREQUIRES(LLFIO_TPRED(is_source_acceptable<T>)) constexpr int compare(path_view_component p, const std::locale &loc) const @@ -793,7 +793,7 @@ public: }); } //! \overload - LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_c_str_deleter<T[]>, + LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_rendered_path_deleter<T[]>, size_t _internal_buffer_size = default_internal_buffer_size) LLFIO_TREQUIRES(LLFIO_TPRED(is_source_acceptable<T>)) constexpr int compare(path_view_component p) const @@ -804,6 +804,118 @@ public: }); } +private: + template <class T> struct _rendered_path_base_ + { + protected: + using _view_type = span<const T>; + _view_type _ref; + + constexpr basic_string_view<T> _as_string_view() const { return {_ref.data(), _ref.size()}; } + + public: + //! Type of the value type + using value_type = T; + //! Type of the pointer type + using pointer = const T *; + //! Type of the const pointer type + using const_pointer = const T *; + //! Type of the reference type + using reference = const T &; + //! Type of the const reference type + using const_reference = const T &; + //! Type of the iterator type + using iterator = typename _view_type::iterator; + //! Type of the const iterator type + using const_iterator = typename _view_type::const_iterator; + //! Type of the reverse iterator type + using reverse_iterator = typename _view_type::reverse_iterator; + //! Type of the const reverse iterator type + using const_reverse_iterator = typename _view_type::const_reverse_iterator; + //! Type of the size type + using size_type = typename _view_type::size_type; + //! Type of the difference type + using difference_type = typename _view_type::difference_type; + + _rendered_path_base_() = default; + _rendered_path_base_(const _rendered_path_base_ &) = default; + _rendered_path_base_ &operator=(const _rendered_path_base_ &) = default; + ~_rendered_path_base_() = default; + + //! Begin iteration + constexpr iterator begin() { return _ref.begin(); } + //! Begin iteration + constexpr const_iterator begin() const { return _ref.begin(); } + //! Begin iteration + constexpr const_iterator cbegin() const { return _ref.cbegin(); } + //! End iteration + constexpr iterator end() { return _ref.end(); } + //! End iteration + constexpr const_iterator end() const { return _ref.end(); } + //! End iteration + constexpr const_iterator cend() const { return _ref.cend(); } + //! Begin reverse iteration + constexpr reverse_iterator rbegin() { return _ref.rbegin(); } + //! Begin reverse iteration + constexpr const_reverse_iterator rbegin() const { return _ref.rbegin(); } + //! Begin reverse iteration + constexpr const_reverse_iterator crbegin() const { return _ref.crbegin(); } + //! End reverse iteration + constexpr reverse_iterator rend() { return _ref.rend(); } + //! End reverse iteration + constexpr const_reverse_iterator rend() const { return _ref.rend(); } + //! End reverse iteration + constexpr const_reverse_iterator crend() const { return _ref.crend(); } + + //! Access + constexpr reference operator[](size_type idx) { return _ref[idx]; } + //! Access + constexpr const_reference operator[](size_type idx) const { return _ref[idx]; } + //! Access + constexpr reference at(size_type idx) { return _ref.at(idx); } + //! Access + constexpr const_reference at(size_type idx) const { return _ref.at(idx); } + //! Access + constexpr reference front() { return _ref.front(); } + //! Access + constexpr const_reference front() const { return _ref.front(); } + //! Access + constexpr reference back() { return _ref.back(); } + //! Access + constexpr const_reference back() const { return _ref.back(); } + //! Access + constexpr pointer data() { return _ref.data(); } + //! Access + constexpr const_pointer data() const { return _ref.data(); } + //! Size + constexpr size_type size() const { return _ref.size(); } + //! Size + constexpr size_type length() const { return _ref.length(); } + //! Max size + constexpr size_type max_size() const { return _ref.max_size(); } + //! Empty + QUICKCPPLIB_NODISCARD constexpr bool empty() const { return _ref.empty(); } + //! As span + constexpr _view_type as_span() const { return _ref; } + }; + template <enum path_view_component::zero_termination ZeroTermination, class T, bool = false> struct _rendered_path_base : public _rendered_path_base_<T> + { + //! As string view + constexpr basic_string_view<T> as_string_view() const { return {this->_ref.data(), this->_ref.size()}; } + }; + template <class T> struct _rendered_path_base<zero_terminated, T> : public _rendered_path_base_<T> + { + //! Return a zero-terminated character array suitable for use as a C string + constexpr const T *c_str() const noexcept { return this->_ref.data(); } + }; + template <bool _> struct _rendered_path_base<zero_terminated, byte, _> : public _rendered_path_base_<byte> + { + }; + template <bool _> struct _rendered_path_base<not_zero_terminated, byte, _> : public _rendered_path_base_<byte> + { + }; + +public: /*! Instantiate from a `path_view_component` to get a path suitable for feeding to other code. \tparam T The destination encoding required. \tparam Deleter A custom deleter OR STL allocator for any temporary buffer. @@ -820,34 +932,28 @@ public: and the source is not zero terminated, a straight memory copy is performed into the temporary buffer. - `c_str` contains a temporary buffer sized according to the template parameter. Output + `rendered_path` contains a temporary buffer sized according to the template parameter. Output below that amount involves no dynamic memory allocation. Output above that amount calls `operator new[]`. You can use an externally supplied larger temporary buffer to avoid dynamic memory allocation in all situations. */ - LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class AllocatorOrDeleter = default_c_str_deleter<T[]>, - size_t _internal_buffer_size = default_internal_buffer_size) + LLFIO_TEMPLATE(enum path_view_component::zero_termination ZeroTermination, class T = typename filesystem::path::value_type, + class AllocatorOrDeleter = default_rendered_path_deleter<T[]>, size_t _internal_buffer_size = default_internal_buffer_size) LLFIO_TREQUIRES(LLFIO_TPRED(is_source_acceptable<T>)) - struct c_str + class rendered_path : public _rendered_path_base<ZeroTermination, T> { - static_assert(is_source_acceptable<T>, "path_view_component::c_str<T> does not have a T which is one of byte, char, wchar_t, char8_t nor char16_t"); + using _base = _rendered_path_base<ZeroTermination, T>; + static_assert(is_source_acceptable<T>, "path_view_component::rendered_path<T> does not have a T which is one of byte, char, wchar_t, char8_t nor char16_t"); template <class DestT, class _Deleter, size_t _internal_buffer_size_, class Char1T, class Char2T> friend int path_view_component::_compare(basic_string_view<Char1T> a, enum zero_termination a_zt, basic_string_view<Char2T> b, enum zero_termination b_zt, const std::locale *loc) noexcept; - //! Type of the value type - using value_type = T; + public: + using value_type = typename _base::value_type; //! Type of the allocator, or `void` if that was not configured using allocator_type = decltype(detail::is_allocator(std::declval<AllocatorOrDeleter>())); //! Type of the deleter, or `void` if that was not configured using deleter_type = decltype(detail::is_deleter<value_type>(std::declval<AllocatorOrDeleter>())); - //! The size of the internal temporary buffer - static constexpr size_t internal_buffer_size = (_internal_buffer_size == 0) ? 1 : _internal_buffer_size; - - //! Pointer to the possibly-converted path - const value_type *buffer{nullptr}; - //! Number of characters, excluding zero terminating char, at buffer - size_t length{0}; private: template <class X = void> static constexpr bool _is_deleter_based = std::is_void<allocator_type>::value; @@ -856,22 +962,21 @@ public: LLFIO_TEMPLATE(class U, class source_type) LLFIO_TREQUIRES(LLFIO_TPRED(!std::is_same<source_type, value_type>::value)) - void _make_passthrough(path_view_component /*unused*/, enum zero_termination /*unused*/, U & /*unused*/, const source_type * /*unused*/) + void _make_passthrough(path_view_component /*unused*/, U & /*unused*/, const source_type * /*unused*/) { LLFIO_LOG_FATAL(nullptr, "Passthrough to non-identity type ought to never be called!"); abort(); } - template <class U> void _make_passthrough(path_view_component view, enum zero_termination output_zero_termination, U &allocate, const value_type *source) + template <class U> void _make_passthrough(path_view_component view, U &allocate, const value_type *source) { using LLFIO_V2_NAMESPACE::basic_string_view; // If the consuming API is a NT kernel API, and we have / in the path, we shall need to do slash conversion const bool needs_slash_translation = (filesystem::path::preferred_separator != '/') && (view.formatting() == format::auto_format || view.formatting() == format::generic_format) && view._invoke([](auto sv) { return sv.find('/') != _npos; }); - length = view._length; - if(!needs_slash_translation && (output_zero_termination == not_zero_terminated || view._zero_terminated)) + if(!needs_slash_translation && (this->zero_termination() == not_zero_terminated || view._zero_terminated)) { - buffer = source; + _base::_ref = typename _base::_view_type(source, view._length); } else { @@ -891,30 +996,26 @@ public: // Use the internal buffer memcpy(_buffer, source, required_bytes); _buffer[required_length] = 0; - buffer = _buffer; + _base::_ref = typename _base::_view_type(_buffer, view._length); } else { auto *buffer_ = allocate(required_length); - if(nullptr == buffer_) - { - length = 0; - } - else + if(nullptr != buffer_) { _bytes_to_delete = required_bytes; memcpy(buffer_, source, required_bytes); buffer_[required_length] = 0; - buffer = buffer_; + _base::_ref = typename _base::_view_type(_buffer, view._length); } } if(needs_slash_translation) { - basic_string_view<value_type> sv(buffer, required_length); + auto sv = _base::_as_string_view(); auto idx = sv.find('/'); while(idx != _npos) { - const_cast<value_type *>(buffer)[idx] = filesystem::path::preferred_separator; + const_cast<value_type *>(sv.data())[idx] = filesystem::path::preferred_separator; idx = sv.find('/', idx); } } @@ -922,59 +1023,57 @@ public: } public: - constexpr c_str() {} + constexpr rendered_path() {} #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 4127) // conditional expression is constant #endif private: - template <class U> void _init(path_view_component view, enum zero_termination output_zero_termination, const std::locale *loc, U &&allocate) + template <class U> void _init(path_view_component view, const std::locale *loc, U &&allocate) { if(0 == view._length) { _buffer[0] = 0; - buffer = _buffer; - length = 0; + _base::_ref = typename _base::_view_type(_buffer, size_t(0)); return; } if(std::is_same<T, byte>::value || view._passthrough) { - length = view._length; - buffer = (const value_type *) view._bytestr; + _base::_ref = typename _base::_view_type((value_type *) view._bytestr, view._length); return; } if(std::is_same<T, char>::value && view._char) { - _make_passthrough(view, output_zero_termination, allocate, view._charstr); + _make_passthrough(view, allocate, view._charstr); return; } if(std::is_same<T, wchar_t>::value && view._wchar) { - _make_passthrough(view, output_zero_termination, allocate, view._wcharstr); + _make_passthrough(view, allocate, view._wcharstr); return; } if(std::is_same<T, char8_t>::value && view._utf8) { - _make_passthrough(view, output_zero_termination, allocate, view._char8str); + _make_passthrough(view, allocate, view._char8str); return; } if(std::is_same<T, char16_t>::value && view._utf16) { - _make_passthrough(view, output_zero_termination, allocate, view._char16str); + _make_passthrough(view, allocate, view._char16str); return; } #ifdef _WIN32 // On Windows, consider char16_t input equivalent to wchar_t if(std::is_same<T, wchar_t>::value && view._utf16) { - _make_passthrough(view, output_zero_termination, allocate, view._wcharstr); + _make_passthrough(view, allocate, view._wcharstr); return; } #else // On POSIX, consider char8_t input equivalent to char if(std::is_same<T, char>::value && view._utf8) { - _make_passthrough(view, output_zero_termination, allocate, view._charstr); + _make_passthrough(view, allocate, view._charstr); return; } #endif @@ -1010,15 +1109,13 @@ public: if(0 == required_length) { // The internal buffer was sufficient. - buffer = _buffer; - length = end - _buffer; + _base::_ref = typename _base::_view_type(_buffer, end - _buffer); return; } // The internal buffer is too small. Fall back to dynamic allocation. This may throw. auto *allocated_buffer = allocate(required_length); if(nullptr == allocated_buffer) { - length = 0; return; } _bytes_to_delete = required_length * sizeof(value_type); @@ -1027,31 +1124,30 @@ public: end = allocated_buffer + (end - _buffer); if(view._passthrough) { - end = detail::reencode_path_to(length, end, required_length, view._bytestr, view._length, loc); + end = detail::reencode_path_to(view._length, end, required_length, view._bytestr, view._length, loc); } else if(view._char) { - end = detail::reencode_path_to(length, end, required_length, view._charstr, view._length, loc); + end = detail::reencode_path_to(view._length, end, required_length, view._charstr, view._length, loc); } else if(view._wchar) { - end = detail::reencode_path_to(length, end, required_length, view._wcharstr, view._length, loc); + end = detail::reencode_path_to(view._length, end, required_length, view._wcharstr, view._length, loc); } else if(view._utf8) { - end = detail::reencode_path_to(length, end, required_length, view._char8str, view._length, loc); + end = detail::reencode_path_to(view._length, end, required_length, view._char8str, view._length, loc); } else if(view._utf16) { - end = detail::reencode_path_to(length, end, required_length, view._char16str, view._length, loc); + end = detail::reencode_path_to(view._length, end, required_length, view._char16str, view._length, loc); } else { LLFIO_LOG_FATAL(nullptr, "path_view_component::cstr somehow sees no state!"); abort(); } - buffer = allocated_buffer; - length = end - buffer; + _base::_ref = typename _base::_view_type(allocated_buffer, end - allocated_buffer); } #ifdef _MSC_VER #pragma warning(pop) @@ -1121,11 +1217,11 @@ public: } // used by compare() - c_str(path_view_component view, enum zero_termination output_zero_termination, const std::locale *loc) + rendered_path(path_view_component view, const std::locale *loc) : _deleter1(_default_deleter) , _deleter1arg(&_deleter2) { - _init(view, output_zero_termination, loc, _default_allocate<AllocatorOrDeleter>(&_deleter2)); + _init(view, loc, _default_allocate<AllocatorOrDeleter>(&_deleter2)); } public: @@ -1135,7 +1231,7 @@ public: \param loc The locale to use to perform reencoding. \param allocate Either a callable with prototype `value_type *(size_t length)` which is defaulted to `return static_cast<value_type *>(::operator new[](length * sizeof(value_type)));`, - or a `pmr::memory_resource *`. You can return `nullptr` if you wish, the consumer of `c_str` will + or a `pmr::memory_resource *`. You can return `nullptr` if you wish, the consumer of `rendered_path` will see a `buffer` set to `nullptr`. If `loc` is defaulted, and an error occurs during any conversion from UTF-8 or UTF-16, an exception of @@ -1146,91 +1242,87 @@ public: */ LLFIO_TEMPLATE(class U, class V) LLFIO_TREQUIRES(LLFIO_TPRED(_is_deleter_based<U>), LLFIO_TEXPR(std::declval<U>()((size_t) 1)), LLFIO_TEXPR(std::declval<V>()((value_type *) nullptr))) - c_str(path_view_component view, enum zero_termination output_zero_termination, const std::locale &loc, U &&allocate, V &&deleter = AllocatorOrDeleter(), - _custom_callable_deleter_tag = {}) + rendered_path(path_view_component view, const std::locale &loc, U &&allocate, V &&deleter = AllocatorOrDeleter(), _custom_callable_deleter_tag = {}) : _deleter1(_invoke_deleter) , _deleter1arg(&_deleter2) , _deleter2(static_cast<V &&>(deleter)) { - _init(view, output_zero_termination, loc, static_cast<U &&>(allocate)); + _init(view, loc, static_cast<U &&>(allocate)); } //! \overload LLFIO_TEMPLATE(class U, class V) LLFIO_TREQUIRES(LLFIO_TPRED(_is_deleter_based<U>), LLFIO_TEXPR(std::declval<U>()((size_t) 1)), LLFIO_TEXPR(std::declval<V>()((value_type *) nullptr))) - c_str(path_view_component view, enum zero_termination output_zero_termination, U &&allocate, V &&deleter = AllocatorOrDeleter(), - _custom_callable_deleter_tag = {}) + rendered_path(path_view_component view, U &&allocate, V &&deleter = AllocatorOrDeleter(), _custom_callable_deleter_tag = {}) : _deleter1(_invoke_deleter) , _deleter1arg(&_deleter2) , _deleter2(static_cast<V &&>(deleter)) { - _init(view, output_zero_termination, (const std::locale *) nullptr, static_cast<U &&>(allocate)); + _init(view, (const std::locale *) nullptr, static_cast<U &&>(allocate)); } //! \overload memory_resource - c_str(path_view_component view, enum zero_termination output_zero_termination, const std::locale &loc, pmr::memory_resource &mr, _memory_resource_tag = {}) + rendered_path(path_view_component view, const std::locale &loc, pmr::memory_resource &mr, _memory_resource_tag = {}) : _deleter1(_memory_resouce_deallocate) , _deleter1arg(&mr) { - _init(view, output_zero_termination, &loc, _memory_resource_allocate{&mr}); + _init(view, &loc, _memory_resource_allocate{&mr}); } //! \overload memory_resource - c_str(path_view_component view, enum zero_termination output_zero_termination, pmr::memory_resource &mr, _memory_resource_tag = {}) + rendered_path(path_view_component view, pmr::memory_resource &mr, _memory_resource_tag = {}) : _deleter1(_memory_resouce_deallocate) , _deleter1arg(&mr) { - _init(view, output_zero_termination, (const std::locale *) nullptr, _memory_resource_allocate{&mr}); + _init(view, (const std::locale *) nullptr, _memory_resource_allocate{&mr}); } //! \overload STL allocator LLFIO_TEMPLATE(class U) LLFIO_TREQUIRES(LLFIO_TPRED(_is_allocator_based<U>), LLFIO_TEXPR(std::declval<U>().allocate((size_t) 1))) - c_str(path_view_component view, enum zero_termination output_zero_termination, const std::locale &loc, U &&allocate, _stl_allocator_tag = {}) + rendered_path(path_view_component view, const std::locale &loc, U &&allocate, _stl_allocator_tag = {}) : _deleter1(_stl_allocator_deallocate<allocator_type>) , _deleter1arg(&_deleter2) , _deleter2(static_cast<U &&>(allocate)) { - _init(view, output_zero_termination, &loc, _stl_allocator_allocate<std::decay_t<U>>(static_cast<std::decay_t<U> *>(&_deleter2))); + _init(view, &loc, _stl_allocator_allocate<std::decay_t<U>>(static_cast<std::decay_t<U> *>(&_deleter2))); } //! \overload STL allocator LLFIO_TEMPLATE(class U) LLFIO_TREQUIRES(LLFIO_TPRED(_is_allocator_based<U>), LLFIO_TEXPR(std::declval<U>().allocate((size_t) 1))) - c_str(path_view_component view, enum zero_termination output_zero_termination, U &&allocate, _stl_allocator_tag = {}) + rendered_path(path_view_component view, U &&allocate, _stl_allocator_tag = {}) : _deleter1(_stl_allocator_deallocate<allocator_type>) , _deleter1arg(&_deleter2) , _deleter2(static_cast<U &&>(allocate)) { - _init(view, output_zero_termination, (const std::locale *) nullptr, _stl_allocator_allocate<std::decay_t<U>>(static_cast<std::decay_t<U> *>(&_deleter2))); + _init(view, (const std::locale *) nullptr, _stl_allocator_allocate<std::decay_t<U>>(static_cast<std::decay_t<U> *>(&_deleter2))); } //! \overload default allocation - c_str(path_view_component view, enum zero_termination output_zero_termination, const std::locale &loc) + rendered_path(path_view_component view, const std::locale &loc) : _deleter1(_default_deleter) , _deleter1arg(&_deleter2) { - _init(view, output_zero_termination, &loc, _default_allocate<AllocatorOrDeleter>(&_deleter2)); + _init(view, &loc, _default_allocate<AllocatorOrDeleter>(&_deleter2)); } //! \overload - c_str(path_view_component view, enum zero_termination output_zero_termination) + explicit rendered_path(path_view_component view) : _deleter1(_default_deleter) , _deleter1arg(&_deleter2) { - _init(view, output_zero_termination, nullptr, _default_allocate<AllocatorOrDeleter>(&_deleter2)); + _init(view, nullptr, _default_allocate<AllocatorOrDeleter>(&_deleter2)); } - //! Construct from a compatible `c_str`. + //! Construct from a compatible `rendered_path`. LLFIO_TEMPLATE(class AllocatorOrDeleter2, size_t _internal_buffer_size2) - LLFIO_TREQUIRES(LLFIO_TPRED(!std::is_same<c_str, c_str<T, AllocatorOrDeleter2, _internal_buffer_size2>>::value), + LLFIO_TREQUIRES(LLFIO_TPRED(!std::is_same<rendered_path, rendered_path<ZeroTermination, T, AllocatorOrDeleter2, _internal_buffer_size2>>::value), LLFIO_TPRED(std::is_constructible<AllocatorOrDeleter, AllocatorOrDeleter2>::value)) - explicit c_str(c_str<T, AllocatorOrDeleter2, _internal_buffer_size2> &&o) noexcept - : buffer(o.buffer) - , length(o.length) + explicit rendered_path(rendered_path<ZeroTermination, T, AllocatorOrDeleter2, _internal_buffer_size2> &&o) noexcept + : _base(o) , _bytes_to_delete(o._bytes_to_delete) , _deleter1(o._deleter1) , _deleter1arg(o._deleter1arg) , _deleter2(std::move(o._deleter2)) { } - ~c_str() { reset(); } - c_str(const c_str &) = delete; - c_str(c_str &&o) noexcept - : buffer(o.buffer) - , length(o.length) + ~rendered_path() { reset(); } + rendered_path(const rendered_path &) = delete; + rendered_path(rendered_path &&o) noexcept + : _base(o) , _bytes_to_delete(o._bytes_to_delete) , _deleter1(o._deleter1) , _deleter1arg(o._deleter1arg) @@ -1238,30 +1330,30 @@ public: { if(this != &o) { - if(o.buffer == o._buffer) + if(o.data() == o._buffer) { - memcpy(_buffer, o._buffer, (o.length + 1) * sizeof(value_type)); - buffer = _buffer; + memcpy(_buffer, o._buffer, (o.size() + 1) * sizeof(value_type)); + _base::_ref = typename _base::_view_type(_buffer, _base::_ref.size()); } if(o._deleter1arg == &o._deleter2) { _deleter1arg = &_deleter2; } - o.buffer = nullptr; + o._ref = {}; o._bytes_to_delete = 0; o._deleter1 = nullptr; o._deleter1arg = nullptr; } } - c_str &operator=(const c_str &) = delete; - c_str &operator=(c_str &&o) noexcept + rendered_path &operator=(const rendered_path &) = delete; + rendered_path &operator=(rendered_path &&o) noexcept { if(this == &o) { return *this; } - this->~c_str(); - new(this) c_str(std::move(o)); + this->~rendered_path(); + new(this) rendered_path(std::move(o)); return *this; } @@ -1270,9 +1362,8 @@ public: { if(_bytes_to_delete > 0) { - _deleter1(_deleter1arg, const_cast<value_type *>(buffer), _bytes_to_delete); - buffer = nullptr; - length = 0; + _deleter1(_deleter1arg, const_cast<value_type *>(_base::_ref.data()), _bytes_to_delete); + _base::_ref = typename _base::_view_type{}; _bytes_to_delete = 0; } } @@ -1281,15 +1372,21 @@ public: { if(_bytes_to_delete > 0) { - auto *ret = buffer; - buffer = nullptr; - length = 0; + auto *ret = _base::_ref.data(); + _base::_ref = typename _base::_view_type{}; _bytes_to_delete = 0; return ret; } return nullptr; } + //! The zero termination of this rendered path + static constexpr enum zero_termination zero_termination() noexcept { return ZeroTermination; } + //! The size of the internal buffer + static constexpr size_t internal_buffer_size() noexcept { return (_internal_buffer_size > 0) ? _internal_buffer_size : 1; } + //! The storage capacity, which may be larger than `size()` if the internal buffer is in use + size_t capacity() noexcept { return (this->data() == _buffer) ? internal_buffer_size() : this->size(); } + //! Access the custom deleter instance passed to the constructor const AllocatorOrDeleter &deleter() const noexcept { return _deleter2; } //! Access the custom deleter instance passed to the constructor @@ -1310,17 +1407,26 @@ public: AllocatorOrDeleter _deleter2; // MAKE SURE this is the final item in storage, the compiler will elide the storage // under optimisation if it can prove it is never used. - value_type _buffer[internal_buffer_size]{}; + value_type _buffer[internal_buffer_size()]{}; }; + //! Convenience type alias + template <class T = typename filesystem::path::value_type, class AllocatorOrDeleter = default_rendered_path_deleter<T[]>, + size_t _internal_buffer_size = default_internal_buffer_size> + using zero_terminated_rendered_path = rendered_path<zero_termination::zero_terminated, T, AllocatorOrDeleter, _internal_buffer_size>; + //! Convenience type alias + template <class T = typename filesystem::path::value_type, class AllocatorOrDeleter = default_rendered_path_deleter<T[]>, + size_t _internal_buffer_size = default_internal_buffer_size> + using not_zero_terminated_rendered_path = rendered_path<zero_termination::not_zero_terminated, T, AllocatorOrDeleter, _internal_buffer_size>; #ifdef __cpp_concepts - template <class T, class Deleter, size_t _internal_buffer_size> + template <enum path_view_component::zero_termination ZeroTermination, class T, class Deleter, size_t _internal_buffer_size> requires(is_source_acceptable<T>) #elif defined(_MSC_VER) - template <class T, class Deleter, size_t _internal_buffer_size, class> + template <enum path_view_component::zero_termination ZeroTermination, class T, class Deleter, size_t _internal_buffer_size, class> #else - template <class T, class Deleter, size_t _internal_buffer_size, typename std::enable_if<(is_source_acceptable<T>), bool>::type> + template <enum path_view_component::zero_termination ZeroTermination, class T, class Deleter, size_t _internal_buffer_size, + typename std::enable_if<(is_source_acceptable<T>), bool>::type> #endif - friend struct c_str; + friend class rendered_path; }; static_assert(std::is_trivially_copyable<path_view_component>::value, "path_view_component is not trivially copyable!"); static_assert(sizeof(path_view_component) == 3 * sizeof(void *), "path_view_component is not three pointers in size!"); @@ -2041,22 +2147,22 @@ public: If the source encodings of the two path views are compatible, a lexicographical comparison is performed. If they are incompatible, either or both views are converted to the destination encoding - using `c_str<T, Delete, _internal_buffer_size>`, and then a + using `rendered_path<T, Delete, _internal_buffer_size>`, and then a lexicographical comparison is performed. This can, for obvious reasons, be expensive. It can also throw - exceptions, as `c_str` does. + exceptions, as `rendered_path` does. If the destination encoding is `byte`, `memcmp()` is used, - and `c_str` is never invoked as the two sources are byte + and `rendered_path` is never invoked as the two sources are byte compared directly. */ - LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_c_str_deleter<T[]>, + LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_rendered_path_deleter<T[]>, size_t _internal_buffer_size = default_internal_buffer_size) LLFIO_TREQUIRES(LLFIO_TPRED(path_view::is_source_acceptable<T>)) constexpr inline int compare(path_view p, const std::locale &loc) const; //! \overload - LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_c_str_deleter<T[]>, + LLFIO_TEMPLATE(class T = typename filesystem::path::value_type, class Deleter = default_rendered_path_deleter<T[]>, size_t _internal_buffer_size = default_internal_buffer_size) LLFIO_TREQUIRES(LLFIO_TPRED(path_view::is_source_acceptable<T>)) constexpr inline int compare(path_view p) const; @@ -2385,16 +2491,16 @@ constexpr inline int path_view::compare(path_view o) const //! Append a path view component to a path inline filesystem::path &operator+=(filesystem::path &a, path_view_component b) { - path_view_component::c_str<> zpath(b, path_view_component::not_zero_terminated); - basic_string_view<filesystem::path::value_type> _(zpath.buffer, zpath.length); + path_view_component::not_zero_terminated_rendered_path<> zpath(b); + basic_string_view<filesystem::path::value_type> _ = zpath.as_string_view(); a.concat(_.begin(), _.end()); return a; } //! Append a path view component to a path inline filesystem::path &operator/=(filesystem::path &a, path_view_component b) { - path_view_component::c_str<> zpath(b, path_view_component::not_zero_terminated); - basic_string_view<filesystem::path::value_type> _(zpath.buffer, zpath.length); + path_view_component::not_zero_terminated_rendered_path<> zpath(b); + basic_string_view<filesystem::path::value_type> _ = zpath.as_string_view(); a.append(_.begin(), _.end()); return a; } diff --git a/include/llfio/v2.0/utils.hpp b/include/llfio/v2.0/utils.hpp index 1877d43a..0abec3e8 100644 --- a/include/llfio/v2.0/utils.hpp +++ b/include/llfio/v2.0/utils.hpp @@ -205,11 +205,31 @@ namespace utils total_address_space_paged_in = 1U << 1U, private_committed = 1U << 2U, private_paged_in = 1U << 3U, + private_committed_inaccurate = 1U << 8U, - all = (unsigned) -1 // + system_physical_memory_total = 1U << 16U, + system_physical_memory_available = 1U << 17U, + system_commit_charge_maximum = 1U << 18U, + system_commit_charge_available = 1U << 19U, + + this_process = 0x0000ffff, // + this_system = 0xffff0000, // + all = 0xffffffff // } QUICKCPPLIB_BITFIELD_END(want) + //! The total physical memory in this system. + uint64_t system_physical_memory_total{0}; + //! The physical memory in this system not containing dirty pages i.e. is currently used for file system caching, or unused. + uint64_t system_physical_memory_available{0}; + + //! The maximum amount of memory which can be committed by all processes. This is typically physical RAM plus swap files. Note that swap files can be added + //! and removed over time. + uint64_t system_commit_charge_maximum{0}; + //! The amount of commit charge remaining before the maximum. Subtract this from `system_commit_charge_maximum` to determine the amount of commit charge + //! consumed by all processes in the system. + uint64_t system_commit_charge_available{0}; + //! The total virtual address space in use. size_t total_address_space_in_use{0}; //! The total memory currently paged into the process. Always `<= total_address_space_in_use`. Also known as "working set", or "resident set size including @@ -233,14 +253,24 @@ namespace utils `process_memory_usage::want::private_committed_inaccurate` can yield significant performance gains. If you set `process_memory_usage::want::private_committed_inaccurate`, we use `/proc/pid/smaps_rollup` and `/proc/pid/maps` to calculate the results. This - cannot distinguish between regions with the accounted - flag enabled or disabled. By default, this fast path is enabled. + cannot distinguish between regions with the accounted flag enabled or disabled, and + be aware that glibc's `malloc()` for some inexplicable reason doesn't set the + accounted flag on regions it commits, so the inaccurate flag will always yield + higher numbers for private commited on Linux. By default, this fast path is enabled. + + \note `/proc/pid/smaps_rollup` was added in Linux kernel 3.16, so the default specifying + `process_memory_usage::want::private_committed_inaccurate` will always fail on Linux + kernels preceding that with an error code comparing equal to `errc::operation_not_supported`. + As one would assume users would prefer this operation to fail on older kernels rather than + silently go slowly in complex memory spaces, it is left opt-in to request + the accurate implementation which works on older Linux kernels. Or, just don't request + `private_committed` at all, and pretend `private_paged_in` means the same thing. \note Mac OS provides no way of reading how much memory a process has committed. We therefore supply as `private_committed` the same value as `private_paged_in`. */ LLFIO_HEADERS_ONLY_FUNC_SPEC result<process_memory_usage> - current_process_memory_usage(process_memory_usage::want want = process_memory_usage::want::all) noexcept; + current_process_memory_usage(process_memory_usage::want want = process_memory_usage::want::this_process) noexcept; /*! \brief CPU usage statistics for a process. */ diff --git a/programs/key-value-store/include/key_value_store.hpp b/programs/key-value-store/include/key_value_store.hpp index 99aef4a0..bd8082a4 100644 --- a/programs/key-value-store/include/key_value_store.hpp +++ b/programs/key-value-store/include/key_value_store.hpp @@ -128,11 +128,12 @@ namespace key_value_store struct index { - uint64_t magic; // versionmagic, currently "AFIOKV01" for valid, "DEADKV01" for requires repair - std::atomic<uint64_t> transaction_counter; // top 16 bits are number of keys changed this transaction, bottom 48 bits are monotonic counter - uint128 hash; // Optional hash of index file written on last close to guard against systems which don't write mmaps properly + uint64_t magic; // versionmagic, currently "AFIOKV01" for valid, "DEADKV01" for requires repair + std::atomic<uint64_t> transaction_counter; // top 16 bits are number of keys changed this transaction, bottom 48 bits are monotonic counter + uint128 hash; // Optional hash of index file written on last close to guard against systems which don't write mmaps properly std::atomic<unsigned> writes_occurring[48]; // Incremented just before an update, decremented after, per writer - std::atomic<bool> all_writes_synced; // Set if all writers since the first which has opened this store did so with `O_SYNC` on (i.e. safe during fsck to check small file tails only) + std::atomic<bool> all_writes_synced; // Set if all writers since the first which has opened this store did so with `O_SYNC` on (i.e. safe during fsck to + // check small file tails only) uint64_t contents_hashed : 1; // If records written are hashed and checked on fetch uint64_t key_is_hash_of_value : 1; // On read, check hash of value equals key @@ -146,12 +147,12 @@ namespace key_value_store uint64_t length; // (uint64_t)-1 means key was deleted }; static_assert(sizeof(value_tail) == 48, "value_tail is wrong size"); - } + } // namespace index class transaction; /*! A transactional key-value store. - */ + */ class basic_key_value_store { friend class transaction; @@ -196,7 +197,8 @@ namespace key_value_store for(size_t n = 0; n < 48; n++) { name = std::to_string(n); - auto fh = llfio::file_handle::file(dir, name, smallfilemode, llfio::file_handle::creation::open_existing, llfio::file_handle::caching::all, llfio::file_handle::flag::disable_prefetching); + auto fh = llfio::file_handle::file(dir, name, smallfilemode, llfio::file_handle::creation::open_existing, llfio::file_handle::caching::all, + llfio::file_handle::flag::disable_prefetching); if(fh) { retry: @@ -207,7 +209,8 @@ namespace key_value_store auto smallfileclaimed = fh.value().lock_file_range(_indexinuseoffset, 1, llfio::lock_kind::exclusive, std::chrono::seconds(0)); if(smallfileclaimed) { - _mysmallfile = llfio::file_handle::file(dir, name, llfio::file_handle::mode::write, llfio::file_handle::creation::open_existing, caching).value(); + _mysmallfile = + llfio::file_handle::file(dir, name, llfio::file_handle::mode::write, llfio::file_handle::creation::open_existing, caching).value(); _mysmallfile.set_append_only(true).value(); _smallfileguard = std::move(smallfileclaimed).value(); _mysmallfileidx = n; @@ -220,7 +223,8 @@ namespace key_value_store { #ifndef _WIN32 // We really need this to only have read only perms, otherwise any mmaps will extend the file ludicrously - fh = llfio::file_handle::file(dir, name, llfio::file_handle::mode::read, llfio::file_handle::creation::open_existing, llfio::file_handle::caching::all, llfio::file_handle::flag::disable_prefetching); + fh = llfio::file_handle::file(dir, name, llfio::file_handle::mode::read, llfio::file_handle::creation::open_existing, + llfio::file_handle::caching::all, llfio::file_handle::flag::disable_prefetching); #endif _smallfiles.blocking.push_back(std::move(fh).value()); } @@ -244,7 +248,9 @@ namespace key_value_store throw maximum_writers_reached(); } // Set up the index, either r/w or read only with copy on write - llfio::section_handle::flag mapflags = (mode == llfio::file_handle::mode::write) ? llfio::section_handle::flag::readwrite : (llfio::section_handle::flag::read | llfio::section_handle::flag::cow); + llfio::section_handle::flag mapflags = (mode == llfio::file_handle::mode::write) ? + llfio::section_handle::flag::readwrite : + (llfio::section_handle::flag::read | llfio::section_handle::flag::cow); llfio::section_handle sh = llfio::section_handle::section(_indexfile, 0, mapflags).value(); llfio::file_handle::extent_type len = sh.length().value(); len -= sizeof(index::index); @@ -266,8 +272,14 @@ namespace key_value_store basic_key_value_store &operator=(const basic_key_value_store &) = delete; basic_key_value_store &operator=(basic_key_value_store &&) = delete; - basic_key_value_store(const llfio::path_handle &dir, size_t hashtableentries, bool enable_integrity = false, llfio::file_handle::mode mode = llfio::file_handle::mode::write, llfio::file_handle::caching caching = llfio::file_handle::caching::all) - : _indexfile(llfio::file_handle::file(dir, "index", mode, (mode == llfio::file_handle::mode::write) ? llfio::file_handle::creation::if_needed : llfio::file_handle::creation::open_existing, caching, llfio::file_handle::flag::disable_prefetching).value()) + basic_key_value_store(const llfio::path_handle &dir, size_t hashtableentries, bool enable_integrity = false, + llfio::file_handle::mode mode = llfio::file_handle::mode::write, + llfio::file_handle::caching caching = llfio::file_handle::caching::all) + : _indexfile(llfio::file_handle::file(dir, "index", mode, + (mode == llfio::file_handle::mode::write) ? llfio::file_handle::creation::if_needed : + llfio::file_handle::creation::open_existing, + caching, llfio::file_handle::flag::disable_prefetching) + .value()) { if(mode == llfio::file_handle::mode::write) { @@ -330,8 +342,12 @@ namespace key_value_store } } //! \overload - basic_key_value_store(const llfio::path_view &dir, size_t hashtableentries, bool enable_integrity = false, llfio::file_handle::mode mode = llfio::file_handle::mode::write, llfio::file_handle::caching caching = llfio::file_handle::caching::all) - : basic_key_value_store(llfio::directory_handle::directory({}, dir, llfio::directory_handle::mode::write, llfio::directory_handle::creation::if_needed).value(), hashtableentries, enable_integrity, mode, caching) + basic_key_value_store(const llfio::path_view &dir, size_t hashtableentries, bool enable_integrity = false, + llfio::file_handle::mode mode = llfio::file_handle::mode::write, + llfio::file_handle::caching caching = llfio::file_handle::caching::all) + : basic_key_value_store( + llfio::directory_handle::directory({}, dir, llfio::directory_handle::mode::write, llfio::directory_handle::creation::if_needed).value(), + hashtableentries, enable_integrity, mode, caching) { } //! Opens the store for read only access @@ -372,7 +388,8 @@ namespace key_value_store for(size_t n = 0; n < _smallfiles.blocking.size(); n++) { auto currentlength = _smallfiles.blocking[n].maximum_extent().value(); - _smallfiles.mapped.push_back(llfio::mapped_file_handle(std::move(_smallfiles.blocking[n]), (unsigned)(currentlength + overextension))); + _smallfiles.mapped.push_back( + llfio::mapped_file_handle(std::move(_smallfiles.blocking[n]), (unsigned) (currentlength + overextension), llfio::section_handle::flag::none, 0)); } _smallfileguard.set_handle(&_smallfiles.mapped[_mysmallfileidx]); _smallfiles.blocking.clear(); @@ -408,7 +425,14 @@ namespace key_value_store //! When this value was last modified uint64_t transaction_counter; - keyvalue_info(keyvalue_info &&o) noexcept : key(std::move(o.key)), value(std::move(o.value)), transaction_counter(std::move(o.transaction_counter)), _value_buffer(std::move(o._value_buffer)) { o._value_buffer = nullptr; } + keyvalue_info(keyvalue_info &&o) noexcept + : key(std::move(o.key)) + , value(std::move(o.value)) + , transaction_counter(std::move(o.transaction_counter)) + , _value_buffer(std::move(o._value_buffer)) + { + o._value_buffer = nullptr; + } keyvalue_info &operator=(keyvalue_info &&o) noexcept { if(this == &o) @@ -538,7 +562,7 @@ namespace key_value_store }; /*! A transaction object. - */ + */ class transaction { friend class basic_key_value_store; @@ -718,7 +742,8 @@ namespace key_value_store uint64_t this_transaction_counter = 0; { uint64_t old_transaction_counter; - union { + union + { struct { uint64_t values_updated : 16; @@ -732,7 +757,8 @@ namespace key_value_store // Increment bottom 48 bits, letting it wrap if necessary _.counter++; _.values_updated = _items.size(); - } while(!_parent->_indexheader->transaction_counter.compare_exchange_weak(old_transaction_counter, _.this_transaction_counter, std::memory_order_release, std::memory_order_relaxed)); + } while(!_parent->_indexheader->transaction_counter.compare_exchange_weak(old_transaction_counter, _.this_transaction_counter, + std::memory_order_release, std::memory_order_relaxed)); this_transaction_counter = _.this_transaction_counter; } @@ -952,6 +978,6 @@ namespace key_value_store _parent->_indexheader->writes_occurring[_parent->_mysmallfileidx].fetch_sub(1); } }; -} +} // namespace key_value_store #endif diff --git a/test/tests/mapped_file_handle.cpp b/test/tests/mapped_file_handle.cpp new file mode 100644 index 00000000..ad9a4f45 --- /dev/null +++ b/test/tests/mapped_file_handle.cpp @@ -0,0 +1,127 @@ +/* Integration test kernel for whether the mapped file handle works +(C) 2021 Niall Douglas <http://www.nedproductions.biz/> (2 commits) +File Created: Sept 2021 + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License in the accompanying file +Licence.txt or at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Distributed under the Boost Software License, Version 1.0. + (See accompanying file Licence.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + +#include "../test_kernel_decl.hpp" + +static constexpr size_t DATA_SIZE = 1024 * 1024; + +template <class T, class F> void runtest(T &v, F &&write) +{ + namespace llfio = LLFIO_V2_NAMESPACE; + QUICKCPPLIB_NAMESPACE::algorithm::small_prng::small_prng rand; + std::vector<llfio::file_handle::const_buffer_type> buffers; + std::vector<char> bytes(4096); + for(char n = ' '; n < 'z'; n++) + { + memset(bytes.data(), n, bytes.size()); + auto i = rand(); + auto offset = i % DATA_SIZE; + auto bufferslen = (i >> 24); + buffers.resize(bufferslen); + size_t count = 0; + uint32_t b = 0; + for(; b < bufferslen && count < bytes.size() - 15; b++) + { + buffers[b] = llfio::file_handle::const_buffer_type((llfio::byte *) bytes.data() + count, rand() & 15); + count += buffers[b].size(); + } + buffers.resize(b); + llfio::file_handle::io_request<llfio::file_handle::const_buffers_type> req(buffers, offset); + write(v, req); + } +}; + + +static inline void TestMappedFileHandle() +{ + namespace llfio = LLFIO_V2_NAMESPACE; + std::vector<llfio::byte> reference(DATA_SIZE); + runtest(reference, [](std::vector<llfio::byte> &cont, llfio::file_handle::io_request<llfio::file_handle::const_buffers_type> req) { + for(auto &b : req.buffers) + { + memcpy(cont.data() + req.offset, b.data(), b.size()); + req.offset += b.size(); + } + }); + auto mf1 = llfio::mapped_file_handle::mapped_temp_inode(DATA_SIZE).value(); + mf1.truncate(DATA_SIZE).value(); + runtest(mf1, [](llfio::mapped_file_handle &cont, llfio::file_handle::io_request<llfio::file_handle::const_buffers_type> req) { cont.write(req).value(); }); + BOOST_CHECK(0 == memcmp(mf1.address(), reference.data(), DATA_SIZE)); + auto mf2 = + llfio::mapped_file_handle::mapped_temp_inode(DATA_SIZE, llfio::path_discovery::storage_backed_temporary_files_directory(), llfio::file_handle::mode::write, + llfio::file_handle::flag::none, llfio::section_handle::flag::write_via_syscall) + .value(); + mf2.truncate(DATA_SIZE).value(); + runtest(mf2, [](llfio::mapped_file_handle &cont, llfio::file_handle::io_request<llfio::file_handle::const_buffers_type> req) { cont.write(req).value(); }); + BOOST_CHECK(0 == memcmp(mf2.address(), reference.data(), DATA_SIZE)); +} + +static inline void TestMappedFileHandleSubsets() +{ + namespace llfio = LLFIO_V2_NAMESPACE; + auto mf1 = llfio::mapped_file_handle::mapped_temp_inode(DATA_SIZE).value(); + mf1.truncate(DATA_SIZE).value(); + std::vector<llfio::byte> reference(DATA_SIZE); + QUICKCPPLIB_NAMESPACE::algorithm::small_prng::small_prng rand; + for(size_t n = 0; n < DATA_SIZE / 4; n++) + { + ((uint32_t *) mf1.address())[n] = ((uint32_t *) reference.data())[n] = rand(); + } + for(size_t n = 0; n < 1000; n++) + { + auto length = rand() % 1024; + auto offset = rand() % (DATA_SIZE - length); + auto mf2 = mf1.reopen(length, offset).value(); + llfio::mapped_file_handle::buffer_type b(nullptr, DATA_SIZE); + llfio::mapped_file_handle::io_request<llfio::mapped_file_handle::buffers_type> req({&b, 1}, 0); + auto readed = mf2.read(req).value(); + BOOST_CHECK(readed.size() == 1); + BOOST_CHECK(b.size() == length); + auto *shouldbe = mf1.address() + offset; + BOOST_CHECK(0 == memcmp(b.data(), shouldbe, length)); + } + std::vector<char> bytes(4096); + for(char n = ' '; n < 'z'; n++) + { + memset(bytes.data(), n, bytes.size()); + auto length = rand() % 4096; + auto offset = rand() % (DATA_SIZE - length); + auto mf2 = mf1.reopen(length, offset).value(); + llfio::mapped_file_handle::const_buffer_type b((const llfio::byte *) bytes.data(), 4096); + llfio::mapped_file_handle::io_request<llfio::mapped_file_handle::const_buffers_type> req({&b, 1}, 0); + auto written = mf2.write(req).value(); + BOOST_CHECK(written.size() == 1); + BOOST_CHECK(b.size() == length); + auto *shouldbe = mf1.address() + offset; + BOOST_CHECK(0 == memcmp(b.data(), shouldbe, length)); + memcpy(reference.data() + offset, bytes.data(), length); + BOOST_CHECK(0 == memcmp(reference.data(), mf1.address(), DATA_SIZE)); + } +} + + +KERNELTEST_TEST_KERNEL(integration, llfio, mapped_file_handle, cache, "Tests that the mapped_file_handle works as expected", TestMappedFileHandle()) + +KERNELTEST_TEST_KERNEL(integration, llfio, mapped_file_handle, subsets, "Tests that the mapped_file_handle subsets works as expected", + TestMappedFileHandleSubsets()) diff --git a/test/tests/path_view.cpp b/test/tests/path_view.cpp index 6a9f7993..ed6071f0 100644 --- a/test/tests/path_view.cpp +++ b/test/tests/path_view.cpp @@ -130,10 +130,10 @@ static inline void TestPathView() BOOST_CHECK(0 == llfio::path_view("/").without_trailing_separator().compare<>("/")); #ifndef _WIN32 // cstr - llfio::path_view::c_str<> g(e, llfio::path_view::zero_terminated); - BOOST_CHECK(g.buffer != p); // NOLINT - llfio::path_view::c_str<> h(f, llfio::path_view::zero_terminated); - BOOST_CHECK(h.buffer == p + 70); // NOLINT + llfio::path_view::zero_terminated_rendered_path<> g(e); + BOOST_CHECK(g.c_str() != p); // NOLINT + llfio::path_view::zero_terminated_rendered_path<> h(f); + BOOST_CHECK(h.c_str() == p + 70); // NOLINT #endif CheckPathView("/mnt/c/Users/ned/Documents/boostish/afio/programs/build_posix/testdir"); CheckPathView("/mnt/c/Users/ned/Documents/boostish/afio/programs/build_posix/testdir/"); @@ -180,12 +180,12 @@ static inline void TestPathView() #endif BOOST_CHECK(0 == h.compare<>("0")); // cstr - llfio::path_view::c_str<> i(g, llfio::path_view::zero_terminated); - BOOST_CHECK(i.buffer != p2); - llfio::path_view::c_str<> j(g, llfio::path_view::not_zero_terminated); - BOOST_CHECK(j.buffer == p2); - llfio::path_view::c_str<> k(h, llfio::path_view::not_zero_terminated); - BOOST_CHECK(k.buffer == p2 + 70); + llfio::path_view::zero_terminated_rendered_path<> i(g); + BOOST_CHECK(i.c_str() != p2); + llfio::path_view::not_zero_terminated_rendered_path<> j(g); + BOOST_CHECK(j.data() == p2); + llfio::path_view::not_zero_terminated_rendered_path<> k(h); + BOOST_CHECK(k.data() == p2 + 70); CheckPathView(L"\\"); CheckPathView(L"C:\\"); @@ -253,7 +253,7 @@ static inline void TestPathView() void operator()(char * /*unused*/) const { called++; } } deleter{5}; llfio::path_view v("foo", 3, llfio::path_view::not_zero_terminated); - llfio::path_view::c_str<char, custom_delete, 0> zbuff(v, llfio::path_view::zero_terminated, allocator, deleter); + llfio::path_view::zero_terminated_rendered_path<char, custom_delete, 0> zbuff(v, allocator, deleter); zbuff.reset(); BOOST_CHECK(allocator.called == 1); // copy must not be taken BOOST_CHECK(deleter.called == 0); // copy must be taken @@ -275,7 +275,7 @@ static inline void TestPathView() bool do_is_equal(const llfio::pmr::memory_resource & /*unused*/) const noexcept override { return false; } } resource; llfio::path_view v("foo", 3, llfio::path_view::not_zero_terminated); - llfio::path_view::c_str<char, std::default_delete<char[]>, 0> zbuff(v, llfio::path_view::zero_terminated, resource); + llfio::path_view::zero_terminated_rendered_path<char, std::default_delete<char[]>, 0> zbuff(v, resource); zbuff.reset(); BOOST_CHECK(resource.allocated == 1); BOOST_CHECK(resource.deleted == 1); @@ -297,7 +297,7 @@ static inline void TestPathView() void deallocate(void * /*unused*/, size_t /*unused*/) { deleted++; } } allocator{5}; llfio::path_view v("foo", 3, llfio::path_view::not_zero_terminated); - llfio::path_view::c_str<char, custom_allocator, 0> zbuff(v, llfio::path_view::zero_terminated, allocator); + llfio::path_view::zero_terminated_rendered_path<char, custom_allocator, 0> zbuff(v, allocator); zbuff.reset(); BOOST_CHECK(allocator.allocated == 0); // copy must be taken BOOST_CHECK(allocator.deleted == 0); // copy must be taken @@ -321,7 +321,7 @@ static inline void TestPathView() void deallocate(void * /*unused*/, size_t /*unused*/) { deleted++; } } allocator{5}; llfio::path_view v("foo", 3, llfio::path_view::not_zero_terminated); - llfio::path_view::c_str<char, custom_allocator, 0> zbuff(v, llfio::path_view::zero_terminated); + llfio::path_view::zero_terminated_rendered_path<char, custom_allocator, 0> zbuff(v); zbuff.reset(); BOOST_CHECK(allocator.allocated == 0); // copy must be taken BOOST_CHECK(allocator.deleted == 0); // copy must be taken diff --git a/test/tests/utils.cpp b/test/tests/utils.cpp index 82967e87..13ab969f 100644 --- a/test/tests/utils.cpp +++ b/test/tests/utils.cpp @@ -51,10 +51,9 @@ static inline void TestCurrentProcessCPUUsage() #ifndef __APPLE__ // On Mac CI at least, idle is approx 2x user which ought to not occur on a two CPU VM BOOST_CHECK(diff.system_ns_in_idle_mode <= 1100000000ULL * thread_count); #endif - std::cout << "With " << thread_count << " threads busy the process spent " << diff.process_ns_in_user_mode - << " ns in user mode and " << diff.process_ns_in_kernel_mode << " ns in kernel mode. The system spent " << diff.system_ns_in_user_mode - << " ns in user mode, " << diff.system_ns_in_kernel_mode << " ns in kernel mode, and " << diff.system_ns_in_idle_mode << " in idle mode." - << std::endl; + std::cout << "With " << thread_count << " threads busy the process spent " << diff.process_ns_in_user_mode << " ns in user mode and " + << diff.process_ns_in_kernel_mode << " ns in kernel mode. The system spent " << diff.system_ns_in_user_mode << " ns in user mode, " + << diff.system_ns_in_kernel_mode << " ns in kernel mode, and " << diff.system_ns_in_idle_mode << " in idle mode." << std::endl; done = true; for(auto &t : threads) { @@ -72,11 +71,12 @@ static inline void TestCurrentProcessMemoryUsage() namespace llfio = LLFIO_V2_NAMESPACE; static const llfio::utils::process_memory_usage *last_pmu; auto print = [](llfio::utils::process_memory_usage &pmu) -> std::ostream &(*) (std::ostream &) { - pmu = llfio::utils::current_process_memory_usage().value(); + pmu = llfio::utils::current_process_memory_usage(llfio::utils::process_memory_usage::want::all).value(); last_pmu = &pmu; return [](std::ostream &s) -> std::ostream & { return s << " " << (last_pmu->total_address_space_in_use / 1024.0 / 1024.0) << "," << (last_pmu->total_address_space_paged_in / 1024.0 / 1024.0) - << "," << (last_pmu->private_committed / 1024.0 / 1024.0) << "," << (last_pmu->private_paged_in / 1024.0 / 1024.0) << std::endl; + << "," << (last_pmu->private_committed / 1024.0 / 1024.0) << "," << (last_pmu->private_paged_in / 1024.0 / 1024.0) << "," + << ((last_pmu->system_commit_charge_maximum - last_pmu->system_commit_charge_available) / 1024.0 / 1024.0) << std::endl; }; }; auto fakefault = [](llfio::map_handle &maph) { @@ -108,7 +108,7 @@ static inline void TestCurrentProcessMemoryUsage() std::cout << "For page allocation:\n"; { llfio::utils::process_memory_usage before_anything, after_reserve, after_commit, after_fault, after_decommit, after_zero, after_do_not_store; - std::cout << " Total address space, Total address space paged in, Private bytes committed, Private bytes paged in\n\n"; + std::cout << " Total address space, Total address space paged in, Private bytes committed, Private bytes paged in, System commit charge\n\n"; std::cout << " Before anything:\n" << print(before_anything) << std::endl; { // Should raise total_address_space_in_use by 1Gb @@ -148,35 +148,41 @@ static inline void TestCurrentProcessMemoryUsage() std::cout << " After committing and faulting and do not storing 1Gb:\n" << print(after_do_not_store) << std::endl; } auto within = [](const llfio::utils::process_memory_usage &a, const llfio::utils::process_memory_usage &b, int total_address_space_in_use, - int total_address_space_paged_in, int private_committed, int private_paged_in) { + int total_address_space_paged_in, int private_committed, int private_paged_in, int system_committed) { auto check = [](size_t a, size_t b, int bound) { - auto diff = abs((b / 1024.0 / 1024.0) - (a / 1024.0 / 1024.0)); - return diff > bound - 10 && diff < bound + 10; + if(bound == INT_MAX) + { + return true; + } + auto diff = (b / 1024.0 / 1024.0) - (a / 1024.0 / 1024.0); + return diff > bound - 50 && diff < bound + 50; }; return check(a.total_address_space_in_use, b.total_address_space_in_use, total_address_space_in_use) // && check(a.total_address_space_paged_in, b.total_address_space_paged_in, total_address_space_paged_in) // && check(a.private_committed, b.private_committed, private_committed) // - && check(a.private_paged_in, b.private_paged_in, private_paged_in); + && check(a.private_paged_in, b.private_paged_in, private_paged_in) // + && check(b.system_commit_charge_available, a.system_commit_charge_available, system_committed); }; #ifdef __APPLE__ // Mac OS has no way of differentiating between committed and paged in pages :( - BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 1024, 1024)); - BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_zero, 1024, 1024, 0, 0)); - BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 1024, 1024)); // do_not_store() doesn't decrease RSS + BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 1024, 1024, 0)); + BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_zero, 1024, 1024, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 1024, 1024, 0)); // do_not_store() doesn't decrease RSS #else - BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 1024, 0)); - BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 1024, 1024)); - BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 1024, 0, 1024)); + BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 1024, 1024, 1024)); + BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0, 0)); #ifdef _WIN32 - BOOST_CHECK(within(before_anything, after_zero, 1024, 0, 1024, 0)); - BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 0, 1024, 0)); // do_not_store() decreases RSS but not commit on Windows + BOOST_CHECK(within(before_anything, after_zero, 1024, 0, 1024, 0, 1024)); + BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 0, 1024, 0, 1024)); // do_not_store() decreases RSS but not commit on Windows #else (void) after_zero; // may not evict faulted set on POSIX - BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 0, 1024)); // do_not_store() decreases commit but does not RSS on POSIX + BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 0, 1024, + INT_MAX)); // do_not_store() decreases commit but does not RSS on POSIX BUT not the system commit if Linux < 4.12 #endif #endif } @@ -189,7 +195,7 @@ static inline void TestCurrentProcessMemoryUsage() { auto sectionh = llfio::section_handle::section(1024 * 1024 * 1024).value(); llfio::utils::process_memory_usage before_anything, after_reserve, after_commit, after_fault, after_decommit, after_zero, after_do_not_store; - std::cout << " Total address space, Total address space paged in, Private bytes committed, Private bytes paged in\n\n"; + std::cout << " Total address space, Total address space paged in, Private bytes committed, Private bytes paged in, System commit charge\n\n"; std::cout << " Before anything:\n" << print(before_anything) << std::endl; { @@ -236,32 +242,37 @@ static inline void TestCurrentProcessMemoryUsage() } #endif auto within = [](const llfio::utils::process_memory_usage &a, const llfio::utils::process_memory_usage &b, int total_address_space_in_use, - int total_address_space_paged_in, int private_committed, int private_paged_in) { + int total_address_space_paged_in, int private_committed, int private_paged_in, int system_committed) { auto check = [](size_t a, size_t b, int bound) { - auto diff = abs((b / 1024.0 / 1024.0) - (a / 1024.0 / 1024.0)); - return diff > bound - 10 && diff < bound + 10; + if(bound == INT_MAX) + { + return true; + } + auto diff = (b / 1024.0 / 1024.0) - (a / 1024.0 / 1024.0); + return diff > bound - 50 && diff < bound + 50; }; return check(a.total_address_space_in_use, b.total_address_space_in_use, total_address_space_in_use) // && check(a.total_address_space_paged_in, b.total_address_space_paged_in, total_address_space_paged_in) // && check(a.private_committed, b.private_committed, private_committed) // - && check(a.private_paged_in, b.private_paged_in, private_paged_in); + && check(a.private_paged_in, b.private_paged_in, private_paged_in) // + && check(b.system_commit_charge_available, a.system_commit_charge_available, system_committed); }; #ifdef __APPLE__ // Mac OS has no way of differentiating between committed and paged in pages :( - BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 0, 0)); - BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_zero, 1024, 1024, 0, 0)); // doesn't implement zero() - BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 0, 0)); // do_not_store() doesn't decrease RSS + BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_zero, 1024, 1024, 0, 0, 0)); // doesn't implement zero() + BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 1024, 0, 0, 0)); // do_not_store() doesn't decrease RSS #else - BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 0, 0)); + BOOST_CHECK(within(before_anything, after_reserve, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_commit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_fault, 1024, 1024, 0, 0, 0)); #ifndef _WIN32 - BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_zero, 1024, 0, 0, 0)); - BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_decommit, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_zero, 1024, 0, 0, 0, 0)); + BOOST_CHECK(within(before_anything, after_do_not_store, 1024, 0, 0, 0, INT_MAX)); // system commit varies if Linux > 4.12 #else (void) after_decommit; (void) after_zero; |