From be9a125ab6a9eb2ae27ec2730166c9ecf4eafe8f Mon Sep 17 00:00:00 2001 From: "Niall Douglas (s [underscore] sourceforge {at} nedprod [dot] com)" Date: Fri, 20 Aug 2021 19:05:07 +0100 Subject: Using the new bitwise tries algorithm in QuickCppLib, implement map_handle caching. Still need to write a test for this however. --- cmake/headers.cmake | 1 + include/llfio/revision.hpp | 6 +- include/llfio/v2.0/detail/impl/map_handle.ipp | 247 +++++++++++++++++++++ .../llfio/v2.0/detail/impl/posix/map_handle.ipp | 86 ++++--- .../llfio/v2.0/detail/impl/windows/map_handle.ipp | 14 +- include/llfio/v2.0/map_handle.hpp | 175 +++++++++++---- 6 files changed, 455 insertions(+), 74 deletions(-) create mode 100644 include/llfio/v2.0/detail/impl/map_handle.ipp diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 88c7deb9..3ecb486f 100644 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -32,6 +32,7 @@ set(llfio_HEADERS "include/llfio/v2.0/detail/impl/dynamic_thread_pool_group.ipp" "include/llfio/v2.0/detail/impl/fast_random_file_handle.ipp" "include/llfio/v2.0/detail/impl/io_multiplexer.ipp" + "include/llfio/v2.0/detail/impl/map_handle.ipp" "include/llfio/v2.0/detail/impl/path_discovery.ipp" "include/llfio/v2.0/detail/impl/path_view.ipp" "include/llfio/v2.0/detail/impl/posix/directory_handle.ipp" diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp index 57cfc606..809c1844 100644 --- a/include/llfio/revision.hpp +++ b/include/llfio/revision.hpp @@ -1,4 +1,4 @@ // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time -#define LLFIO_PREVIOUS_COMMIT_REF 5544436fd6654fcc6cd6b3a9c3f3b9b7e0b21865 -#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-03 19:34:21 +00:00" -#define LLFIO_PREVIOUS_COMMIT_UNIQUE 5544436f +#define LLFIO_PREVIOUS_COMMIT_REF e43ef4f6953230803a30453bbab20061b009fe05 +#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-20 09:36:50 +00:00" +#define LLFIO_PREVIOUS_COMMIT_UNIQUE e43ef4f6 diff --git a/include/llfio/v2.0/detail/impl/map_handle.ipp b/include/llfio/v2.0/detail/impl/map_handle.ipp new file mode 100644 index 00000000..b4df1724 --- /dev/null +++ b/include/llfio/v2.0/detail/impl/map_handle.ipp @@ -0,0 +1,247 @@ +/* A handle to a source of mapped memory +(C) 2021 Niall Douglas (10 commits) +File Created: Aug 2021 + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License in the accompanying file +Licence.txt or at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Distributed under the Boost Software License, Version 1.0. + (See accompanying file Licence.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + +#include "../../map_handle.hpp" +#include "../../utils.hpp" + +#include +#include + +#include + +LLFIO_V2_NAMESPACE_BEGIN + +namespace detail +{ + struct map_handle_cache_item_t + { + map_handle_cache_item_t *trie_parent{nullptr}; + map_handle_cache_item_t *trie_child[2]; + map_handle_cache_item_t *trie_sibling[2]; + size_t trie_key{0}; // allocation size shifted right by map_handle_cache_page_size_shift + size_t page_size{0}; + void *addr{nullptr}; + std::chrono::steady_clock::time_point when_added{std::chrono::steady_clock::now()}; + + map_handle_cache_item_t(size_t key, size_t _page_size, void *_addr) + : trie_key(key) + , page_size(_page_size) + , addr(_addr) + { + } + }; + struct map_handle_cache_base_t + { + std::mutex lock; + size_t trie_count{0}; + map_handle_cache_item_t *trie_children[8 * sizeof(size_t)]; + bool trie_nobbledir{0}; + size_t bytes_in_cache{0}; + }; + static const size_t page_size_shift = [] { return QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::detail::bitscanr(utils::page_size()); }(); + class map_handle_cache_t : protected QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie + { + using _base = QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie; + using _lock_guard = std::unique_lock; + + public: + std::atomic do_not_store_failed_count{0}; + + using _base::size; + void *get(size_t bytes, size_t page_size) + { + const auto _bytes = bytes >> page_size_shift; + _lock_guard g(lock); + auto it = _base::find_equal_or_larger(_bytes, 7 /* 99% close to the key after page size shift */); + for(; it != _base::end() && page_size != it->page_size && _bytes == it->trie_key; ++it) + { + } + if(it == _base::end() || page_size != it->page_size || _bytes != it->trie_key) + { + return nullptr; + } + auto *p = *it; + _base::erase(it); + _base::bytes_in_cache -= bytes; + assert(bytes == p->trie_key << page_size_shift); + g.unlock(); + void *ret = p->addr; + delete p; + return ret; + } + void add(size_t bytes, size_t page_size, void *addr) + { + const auto _bytes = bytes >> page_size_shift; + auto *p = new map_handle_cache_item_t(_bytes, page_size, addr); + _lock_guard g(lock); + _base::insert(p); + _base::bytes_in_cache += bytes; + } + map_handle::cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than) + { + _lock_guard g(lock); + map_handle::cache_statistics ret; + if(older_than != std::chrono::steady_clock::time_point()) + { + for(auto it = _base::begin(); it != _base::end();) + { + if(it->when_added < older_than) + { + auto *p = *it; + it = _base::erase(it); + const auto _bytes = p->trie_key << page_size_shift; + _base::bytes_in_cache -= _bytes; + ret.bytes_just_trimmed += _bytes; + ret.items_just_trimmed++; + delete p; + } + else + { + ++it; + } + } + } + ret.items_in_cache = _base::size(); + ret.bytes_in_cache = _base::bytes_in_cache; + return ret; + } + }; + extern inline QUICKCPPLIB_SYMBOL_EXPORT map_handle_cache_t &map_handle_cache() + { + static map_handle_cache_t v; + return v; + } +} // namespace detail + +result map_handle::_recycled_map(size_type bytes, section_handle::flag _flag) noexcept +{ + if(bytes == 0u) + { + return errc::argument_out_of_domain; + } + result ret(map_handle(nullptr, _flag)); + native_handle_type &nativeh = ret.value()._v; + OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag)); + bytes = utils::round_up_to_page_size(bytes, pagesize); + LLFIO_LOG_FUNCTION_CALL(&ret); + void *addr = detail::map_handle_cache().get(bytes, pagesize); + if(addr == nullptr) + { + return _new_map(bytes, _flag); + } +#ifdef _WIN32 + { + DWORD allocation = MEM_COMMIT, prot; + size_t commitsize; + OUTCOME_TRY(win32_map_flags(nativeh, allocation, prot, commitsize, true, _flag)); + if(VirtualAlloc(addr, bytes, allocation, prot) == nullptr) + { + return win32_error(); + } + // Windows has no way of getting the kernel to prefault maps on creation, so ... + if(_flag & section_handle::flag::prefault) + { + using namespace windows_nt_kernel; + // Start an asynchronous prefetch, so it might fault the whole lot in at once + buffer_type b{static_cast(addr), bytes}; + (void) prefetch(span(&b, 1)); + volatile auto *a = static_cast(addr); + for(size_t n = 0; n < bytes; n += pagesize) + { + a[n]; + } + } + } +#else + int flags = MAP_FIXED; +#ifdef MAP_POPULATE + if(_flag & section_handle::flag::prefault) + { + flags |= MAP_POPULATE; + } +#endif +#ifdef MAP_PREFAULT_READ + if(_flag & section_handle::flag::prefault) + flags |= MAP_PREFAULT_READ; +#endif + OUTCOME_TRY(do_mmap(nativeh, addr, flags, nullptr, pagesize, bytes, 0, _flag)); +#endif + ret.value()._addr = static_cast(addr); + ret.value()._reservation = bytes; + ret.value()._length = bytes; + ret.value()._pagesize = pagesize; + nativeh._init = -2; // otherwise appears closed + nativeh.behaviour |= native_handle_type::disposition::allocation; + return ret; +} + +bool map_handle::_recycle_map() noexcept +{ + try + { + LLFIO_LOG_FUNCTION_CALL(this); + auto &c = detail::map_handle_cache(); +#ifdef _WIN32 + if(!win32_release_nonfile_allocations(_addr, _length, MEM_DECOMMIT)) + { + return false; + } +#else + if(memory_accounting() == memory_accounting_kind::commit_charge) + { + if(!do_mmap(_v, _addr, MAP_FIXED, nullptr, _pagesize, _length, 0, section_handle::flag::none | section_handle::flag::nocommit)) + { + return false; + } + } + else + { +#ifdef __linux__ + if(c.do_not_store_failed_count.load(std::memory_order_relaxed) < 10) + { + auto r = do_not_store({_addr, _length}); + if(!r) + { + c.do_not_store_failed_count.fetch_add(1, std::memory_order_relaxed); + } + } +#endif + } +#endif + c.add(_length, _pagesize, _addr); + return true; // cached + } + catch(...) + { + return false; + } +} + +map_handle::cache_statistics map_handle::trim_cache(std::chrono::steady_clock::time_point older_than) noexcept +{ + return detail::map_handle_cache().trim_cache(older_than); +} + + +LLFIO_V2_NAMESPACE_END diff --git a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp index 397576a8..f77954ca 100644 --- a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp @@ -1,5 +1,5 @@ /* A handle to a source of mapped memory -(C) 2017-2020 Niall Douglas (10 commits) +(C) 2017-2021 Niall Douglas (10 commits) File Created: Apr 2017 @@ -156,8 +156,10 @@ map_handle::~map_handle() auto ret = map_handle::close(); if(ret.has_error()) { - LLFIO_LOG_FATAL(_v.fd, "map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the 64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " - "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); + LLFIO_LOG_FATAL(_v.fd, + "map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the " + "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to " + "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out."); abort(); } } @@ -173,27 +175,31 @@ result map_handle::close() noexcept OUTCOME_TRYV(map_handle::barrier(barrier_kind::wait_all)); } // printf("%d munmap %p-%p\n", getpid(), _addr, _addr+_reservation); - if(-1 == ::munmap(_addr, _reservation)) + if(_section != nullptr || !_recycle_map()) { -#ifdef LLFIO_DEBUG_LINUX_MUNMAP - int olderrno = errno; - ssize_t bytesread; - // Refresh the /proc file - (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_END); - (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_SET); - char buffer[4096]; - (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, sprintf(buffer, "\n---\nCause of munmap failure by process %d: %d (%s)\n\n", getpid(), olderrno, strerror(olderrno))); - do + if(-1 == ::munmap(_addr, _reservation)) { - bytesread = ::read(llfio_linux_munmap_debug.smaps_fd, buffer, sizeof(buffer)); - if(bytesread > 0) +#ifdef LLFIO_DEBUG_LINUX_MUNMAP + int olderrno = errno; + ssize_t bytesread; + // Refresh the /proc file + (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_END); + (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_SET); + char buffer[4096]; + (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, + sprintf(buffer, "\n---\nCause of munmap failure by process %d: %d (%s)\n\n", getpid(), olderrno, strerror(olderrno))); + do { - (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, bytesread); - } - } while(bytesread > 0); - errno = olderrno; + bytesread = ::read(llfio_linux_munmap_debug.smaps_fd, buffer, sizeof(buffer)); + if(bytesread > 0) + { + (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, bytesread); + } + } while(bytesread > 0); + errno = olderrno; #endif - return posix_error(); + return posix_error(); + } } } // We don't want ~handle() to close our borrowed handle @@ -213,7 +219,8 @@ native_handle_type map_handle::release() noexcept return {}; } -map_handle::io_result map_handle::_do_barrier(map_handle::io_request reqs, barrier_kind kind, deadline d) noexcept +map_handle::io_result map_handle::_do_barrier(map_handle::io_request reqs, barrier_kind kind, + deadline d) noexcept { LLFIO_LOG_FUNCTION_CALL(this); byte *addr = _addr + reqs.offset; @@ -255,8 +262,38 @@ map_handle::io_result map_handle::_do_barrier(ma return {reqs.buffers}; } +#ifdef __linux__ +LLFIO_HEADERS_ONLY_MEMFUNC_SPEC map_handle::memory_accounting_kind map_handle::memory_accounting() noexcept +{ + static memory_accounting_kind v{memory_accounting_kind::unknown}; + if(v != memory_accounting_kind::unknown) + { + return v; + } + int fd = ::open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if(fd != -1) + { + char buffer[8]; + if(::read(fd, buffer, 8) > 0) + { + if(buffer[0] == '2') + { + v = memory_accounting_kind::commit_charge; + } + else + { + v = memory_accounting_kind::over_commit; + } + } + ::close(fd); + } + return v; +} +#endif + -static inline result do_mmap(native_handle_type &nativeh, void *ataddr, int extra_flags, section_handle *section, map_handle::size_type pagesize, map_handle::size_type &bytes, map_handle::extent_type offset, section_handle::flag _flag) noexcept +static inline result do_mmap(native_handle_type &nativeh, void *ataddr, int extra_flags, section_handle *section, map_handle::size_type pagesize, + map_handle::size_type &bytes, map_handle::extent_type offset, section_handle::flag _flag) noexcept { bool have_backing = (section != nullptr); int prot = 0, flags = have_backing ? MAP_SHARED : (MAP_PRIVATE | MAP_ANONYMOUS); @@ -387,9 +424,8 @@ static inline result do_mmap(native_handle_type &nativeh, void *ataddr, return addr; } -result map_handle::map(size_type bytes, bool /*unused*/, section_handle::flag _flag) noexcept +result map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept { - // TODO: Keep a cache of MADV_FREE pages deallocated if(bytes == 0u) { return errc::argument_out_of_domain; @@ -495,7 +531,7 @@ result map_handle::truncate(size_type newsize, bool permi _reservation = _newsize; _length = (length - _offset < newsize) ? (length - _offset) : newsize; // length of backing, not reservation return newsize; -#else // generic POSIX, inefficient +#else // generic POSIX, inefficient byte *addrafter = _addr + _reservation; size_type bytes = newsize - _reservation; extent_type offset = _offset + _reservation; diff --git a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp index f1996484..4a175f89 100644 --- a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp +++ b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp @@ -1,5 +1,5 @@ /* A handle to a source of mapped memory -(C) 2016-2020 Niall Douglas (17 commits) +(C) 2016-2021 Niall Douglas (17 commits) File Created: August 2016 @@ -554,7 +554,10 @@ result map_handle::close() noexcept } else { - OUTCOME_TRYV(win32_release_nonfile_allocations(_addr, _reservation, MEM_RELEASE)); + if(!_recycle_map()) + { + OUTCOME_TRYV(win32_release_nonfile_allocations(_addr, _reservation, MEM_RELEASE)); + } } } // We don't want ~handle() to close our borrowed handle @@ -619,9 +622,12 @@ map_handle::io_result map_handle::_do_barrier(ma } -result map_handle::map(size_type bytes, bool /*unused*/, section_handle::flag _flag) noexcept +result map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept { - // TODO: Keep a cache of DiscardVirtualMemory()/MEM_RESET pages deallocated + if(bytes == 0u) + { + return errc::argument_out_of_domain; + } result ret(map_handle(nullptr, _flag)); native_handle_type &nativeh = ret.value()._v; DWORD allocation = MEM_RESERVE | MEM_COMMIT, prot; diff --git a/include/llfio/v2.0/map_handle.hpp b/include/llfio/v2.0/map_handle.hpp index 69241720..ba6b83ef 100644 --- a/include/llfio/v2.0/map_handle.hpp +++ b/include/llfio/v2.0/map_handle.hpp @@ -1,5 +1,5 @@ /* A handle to a source of mapped memory -(C) 2016-2018 Niall Douglas (14 commits) +(C) 2016-2021 Niall Douglas (14 commits) File Created: August 2016 @@ -55,28 +55,32 @@ public: using size_type = handle::size_type; //! The behaviour of the memory section - QUICKCPPLIB_BITFIELD_BEGIN(flag){none = 0U, //!< No flags - read = 1U << 0U, //!< Memory views can be read - write = 1U << 1U, //!< Memory views can be written - cow = 1U << 2U, //!< Memory views can be copy on written - execute = 1U << 3U, //!< Memory views can execute code - - nocommit = 1U << 8U, //!< Don't allocate space for this memory in the system immediately - prefault = 1U << 9U, //!< Prefault, as if by reading every page, any views of memory upon creation. - executable = 1U << 10U, //!< The backing storage is in fact an executable program binary. - singleton = 1U << 11U, //!< A single instance of this section is to be shared by all processes using the same backing file. - - barrier_on_close = 1U << 16U, //!< Maps of this section, if writable, issue a `barrier()` when destructed blocking until data (not metadata) reaches physical storage. - nvram = 1U << 17U, //!< This section is of non-volatile RAM. - write_via_syscall = 1U << 18U, //!< For file backed maps, `map_handle::write()` is implemented as a `write()` syscall to the file descriptor. This causes the map to be mapped read-only. - - page_sizes_1 = 1U << 24U, //!< Use `utils::page_sizes()[1]` sized pages, or fail. - page_sizes_2 = 2U << 24U, //!< Use `utils::page_sizes()[2]` sized pages, or fail. - page_sizes_3 = 3U << 24U, //!< Use `utils::page_sizes()[3]` sized pages, or fail. - - // NOTE: IF UPDATING THIS UPDATE THE std::ostream PRINTER BELOW!!! - - readwrite = (read | write)}; + QUICKCPPLIB_BITFIELD_BEGIN(flag){ + none = 0U, //!< No flags + read = 1U << 0U, //!< Memory views can be read + write = 1U << 1U, //!< Memory views can be written + cow = 1U << 2U, //!< Memory views can be copy on written + execute = 1U << 3U, //!< Memory views can execute code + + nocommit = 1U << 8U, //!< Don't allocate space for this memory in the system immediately + prefault = 1U << 9U, //!< Prefault, as if by reading every page, any views of memory upon creation. + executable = 1U << 10U, //!< The backing storage is in fact an executable program binary. + singleton = 1U << 11U, //!< A single instance of this section is to be shared by all processes using the same backing file. + + barrier_on_close = + 1U << 16U, //!< Maps of this section, if writable, issue a `barrier()` when destructed blocking until data (not metadata) reaches physical storage. + nvram = 1U << 17U, //!< This section is of non-volatile RAM. + write_via_syscall = + 1U + << 18U, //!< For file backed maps, `map_handle::write()` is implemented as a `write()` syscall to the file descriptor. This causes the map to be mapped read-only. + + page_sizes_1 = 1U << 24U, //!< Use `utils::page_sizes()[1]` sized pages, or fail. + page_sizes_2 = 2U << 24U, //!< Use `utils::page_sizes()[2]` sized pages, or fail. + page_sizes_3 = 3U << 24U, //!< Use `utils::page_sizes()[3]` sized pages, or fail. + + // NOTE: IF UPDATING THIS UPDATE THE std::ostream PRINTER BELOW!!! + + readwrite = (read | write)}; QUICKCPPLIB_BITFIELD_END(flag); protected: @@ -154,7 +158,10 @@ public: \errors Any of the values POSIX dup(), open() or NtCreateSection() can return. */ LLFIO_MAKE_FREE_FUNCTION - static result section(file_handle &backing, extent_type bytes = 0) noexcept { return section(backing, bytes, backing.is_writable() ? (flag::readwrite) : (flag::read)); } + static result section(file_handle &backing, extent_type bytes = 0) noexcept + { + return section(backing, bytes, backing.is_writable() ? (flag::readwrite) : (flag::read)); + } /*! \brief Create a memory section backed by an anonymous, managed file. \param bytes The initial size of this section. Cannot be zero. \param dirh Where to create the anonymous, managed file. @@ -163,7 +170,9 @@ public: \errors Any of the values POSIX dup(), open() or NtCreateSection() can return. */ LLFIO_MAKE_FREE_FUNCTION - static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result section(extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), flag _flag = flag::read | flag::write) noexcept; + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result section(extent_type bytes, + const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), + flag _flag = flag::read | flag::write) noexcept; //! Returns the memory section's flags flag section_flags() const noexcept { return _flag; } @@ -454,7 +463,8 @@ protected: if(section != nullptr) { // Apart from read/write/cow/execute, the section's flags override the map's flags - _flag |= (section->section_flags() & ~(section_handle::flag::read | section_handle::flag::write | section_handle::flag::cow | section_handle::flag::execute)); + _flag |= + (section->section_flags() & ~(section_handle::flag::read | section_handle::flag::write | section_handle::flag::cow | section_handle::flag::execute)); } } @@ -463,7 +473,8 @@ public: constexpr map_handle() {} // NOLINT LLFIO_HEADERS_ONLY_VIRTUAL_SPEC ~map_handle() override; //! Construct an instance managing pages at `addr`, `length`, `pagesize` and `flags` - explicit map_handle(byte *addr, size_type length, size_type pagesize, section_handle::flag flags, section_handle *section = nullptr, extent_type offset = 0) noexcept + explicit map_handle(byte *addr, size_type length, size_type pagesize, section_handle::flag flags, section_handle *section = nullptr, + extent_type offset = 0) noexcept : _section(section) , _addr(addr) , _offset(offset) @@ -526,10 +537,18 @@ public: protected: LLFIO_HEADERS_ONLY_VIRTUAL_SPEC size_t _do_max_buffers() const noexcept override { return 0; } - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_barrier(io_request reqs = io_request(), barrier_kind kind = barrier_kind::nowait_data_only, deadline d = deadline()) noexcept override; + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_barrier(io_request reqs = io_request(), + barrier_kind kind = barrier_kind::nowait_data_only, + deadline d = deadline()) noexcept override; LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_read(io_request reqs, deadline d = deadline()) noexcept override; LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result _do_write(io_request reqs, deadline d = deadline()) noexcept override; + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result _new_map(size_type bytes, section_handle::flag _flag) noexcept; + + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result _recycled_map(size_type bytes, section_handle::flag _flag) noexcept; + + LLFIO_HEADERS_ONLY_MEMFUNC_SPEC bool _recycle_map() noexcept; + public: /*! Map unused memory into view, creating new memory if insufficient unused memory is available (i.e. add the returned memory to the process' commit charge, unless `flag::nocommit` @@ -538,7 +557,9 @@ public: \param bytes How many bytes to map. Typically will be rounded up to a multiple of the page size (see `page_size()`). - \param zeroed Set to true if only all bits zeroed memory is wanted. + \param zeroed Set to true if only all bits zeroed memory is wanted. If this is true, a syscall + is always performed as the kernel probably has zeroed pages ready to go, whereas if false, the + request may be satisfied from a local cache instead. The default is false. \param _flag The permissions with which to map the view. \note On Microsoft Windows this constructor uses the faster `VirtualAlloc()` which creates less @@ -547,10 +568,35 @@ public: the other constructor. This makes available all those very useful VM tricks Windows can do with section mapped memory which `VirtualAlloc()` memory cannot do. + When this kind of map handle is closed, it is added to an internal cache so new map + handle creations of this kind with `zeroed = false` are very quick and avoid a syscall. The + internal cache may return a map slightly bigger than requested. If you wish to always invoke + the syscall, specify `zeroed = true`. + + When maps are added to the internal cache, on all systems except Linux the memory is decommitted + first. This reduces commit charge appropriately, thus only virtual address space remains + consumed. On Linux, if `memory_accounting()` is `memory_accounting_kind::commit_charge`, we also + decommit, however be aware that this can increase the average VMA use count in the Linux kernel, and most + Linux kernels are configured with a very low per-process limit of 64k VMAs (this is easy to raise + using `sysctl -w vm.max_map_count=262144`). Otherwise on Linux to avoid increasing VMA count + we instead mark closed maps as `LazyFree`, which means that their contents can be arbitrarily + disposed of by the Linux kernel as needed, but also allows Linux to coalesce VMAs so the very + low per-process limit is less likely to be exceeded. If the `LazyFree` syscall is not implemented on this + Linux, we do nothing. + + \warning The cache does not self-trim on its own, you MUST call `trim_cache()` to trim allocations of + virtual address (these don't count towards process commit charge, but they do consume address space + and precious VMAs in the Linux kernel). Only on 32 bit processes where virtual address space is limited, or on + Linux where VMAs allocated is considered by the Linux OOM killer, will you need to probably + care much about regular cache trimming. + \errors Any of the values POSIX `mmap()` or `VirtualAlloc()` can return. */ LLFIO_MAKE_FREE_FUNCTION - static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result map(size_type bytes, bool zeroed = false, section_handle::flag _flag = section_handle::flag::readwrite) noexcept; + static inline result map(size_type bytes, bool zeroed = false, section_handle::flag _flag = section_handle::flag::readwrite) noexcept + { + return (zeroed || (_flag & section_handle::flag::nocommit)) ? _new_map(bytes, _flag) : _recycled_map(bytes, _flag); + } /*! Reserve address space within which individual pages can later be committed. Reserved address space is NOT added to the process' commit charge. @@ -566,7 +612,7 @@ public: \errors Any of the values POSIX `mmap()` or `VirtualAlloc()` can return. */ LLFIO_MAKE_FREE_FUNCTION - static inline result reserve(size_type bytes) noexcept { return map(bytes, false, section_handle::flag::none | section_handle::flag::nocommit); } + static inline result reserve(size_type bytes) noexcept { return _new_map(bytes, section_handle::flag::none | section_handle::flag::nocommit); } /*! Create a memory mapped view of a backing storage, optionally reserving additional address space for later growth. @@ -585,7 +631,41 @@ public: \errors Any of the values POSIX `mmap()` or `NtMapViewOfSection()` can return. */ LLFIO_MAKE_FREE_FUNCTION - static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result map(section_handle §ion, size_type bytes = 0, extent_type offset = 0, section_handle::flag _flag = section_handle::flag::readwrite) noexcept; + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result map(section_handle §ion, size_type bytes = 0, extent_type offset = 0, + section_handle::flag _flag = section_handle::flag::readwrite) noexcept; + + //! The kind of memory accounting this system uses + enum class memory_accounting_kind + { + unknown, + /*! This system will not permit more than physical RAM and your swap files to be committed. + On every OS except for Linux, this is always the case. + */ + commit_charge, + /*! This system will permit more memory to be committed than physical RAM and your swap + files, and will terminate your process without warning at some unknown point + if you write into enough of the pages committed. This is typically the default on Linux, + but it can be changed at runtime. + */ + over_commit + }; +#ifdef __linux__ + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC memory_accounting_kind memory_accounting() noexcept; +#else + static memory_accounting_kind memory_accounting() noexcept { return memory_accounting_kind::commit_charge; } +#endif + + //! Statistics about the map handle cache + struct cache_statistics + { + size_t items_in_cache{0}; + size_t bytes_in_cache{0}; + size_t items_just_trimmed{0}; + size_t bytes_just_trimmed{0}; + }; + /*! Get statistics about the map handle cache, optionally trimming the least recently used maps. + */ + static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than = {}) noexcept; //! The memory section this handle is using section_handle *section() const noexcept { return _section; } @@ -718,7 +798,8 @@ public: */ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result do_not_store(buffer_type region) noexcept; - //! Ask the system to begin to asynchronously prefetch the span of memory regions given, returning the regions actually prefetched. Note that on Windows 7 or earlier the system call to implement this was not available, and so you will see an empty span returned. + //! Ask the system to begin to asynchronously prefetch the span of memory regions given, returning the regions actually prefetched. Note that on Windows 7 or + //! earlier the system call to implement this was not available, and so you will see an empty span returned. static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result> prefetch(span regions) noexcept; //! \overload static result prefetch(buffer_type region) noexcept @@ -898,7 +979,8 @@ inline void swap(section_handle &self, section_handle &o) noexcept */ inline result section(file_handle &backing, section_handle::extent_type maximum_size, section_handle::flag _flag) noexcept { - return section_handle::section(std::forward(backing), std::forward(maximum_size), std::forward(_flag)); + return section_handle::section(std::forward(backing), std::forward(maximum_size), + std::forward(_flag)); } /*! \brief Create a memory section backed by a file. \param backing The handle to use as backing storage. @@ -919,7 +1001,8 @@ inline result section(file_handle &backing, section_handle::exte \errors Any of the values POSIX dup(), open() or NtCreateSection() can return. */ -inline result section(section_handle::extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), section_handle::flag _flag = section_handle::flag::read | section_handle::flag::write) noexcept +inline result section(section_handle::extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), + section_handle::flag _flag = section_handle::flag::read | section_handle::flag::write) noexcept { return section_handle::section(std::forward(bytes), std::forward(dirh), std::forward(_flag)); } @@ -952,7 +1035,8 @@ inline result close(map_handle &self) noexcept /*! Create new memory and map it into view. \param bytes How many bytes to create and map. Typically will be rounded up to a multiple of the page size (see `page_size()`) on POSIX, 64Kb on Windows. \param zeroed Set to true if only all bits zeroed memory is wanted. -\param _flag The permissions with which to map the view. `flag::none` can be useful for reserving virtual address space without committing system resources, use commit() to later change availability of memory. +\param _flag The permissions with which to map the view. `flag::none` can be useful for reserving virtual address space without committing system resources, use +commit() to later change availability of memory. \note On Microsoft Windows this constructor uses the faster VirtualAlloc() which creates less versatile page backed memory. If you want anonymous memory allocated from a paging file backed section instead, create a page file backed section and then a mapped view from that using @@ -968,14 +1052,18 @@ inline result map(map_handle::size_type bytes, bool zeroed = false, /*! Create a memory mapped view of a backing storage, optionally reserving additional address space for later growth. \param section A memory section handle specifying the backing storage to use. \param bytes How many bytes to reserve (0 = the size of the section). Rounded up to nearest 64Kb on Windows. -\param offset The offset into the backing storage to map from. Typically needs to be at least a multiple of the page size (see `page_size()`), on Windows it needs to be a multiple of the kernel memory allocation granularity (typically 64Kb). -\param _flag The permissions with which to map the view which are constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use commit() to later change availability of memory. +\param offset The offset into the backing storage to map from. Typically needs to be at least a multiple of the page size (see `page_size()`), on Windows it +needs to be a multiple of the kernel memory allocation granularity (typically 64Kb). \param _flag The permissions with which to map the view which are +constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use +commit() to later change availability of memory. \errors Any of the values POSIX mmap() or NtMapViewOfSection() can return. */ -inline result map(section_handle §ion, map_handle::size_type bytes = 0, map_handle::extent_type offset = 0, section_handle::flag _flag = section_handle::flag::readwrite) noexcept +inline result map(section_handle §ion, map_handle::size_type bytes = 0, map_handle::extent_type offset = 0, + section_handle::flag _flag = section_handle::flag::readwrite) noexcept { - return map_handle::map(std::forward(section), std::forward(bytes), std::forward(offset), std::forward(_flag)); + return map_handle::map(std::forward(section), std::forward(bytes), std::forward(offset), + std::forward(_flag)); } //! The size of the memory map. This is the accessible size, NOT the reservation size. inline map_handle::size_type length(const map_handle &self) noexcept @@ -1024,7 +1112,8 @@ The size of each scatter-gather buffer is updated with the number of bytes of th \errors None, though the various signals and structured exception throws common to using memory maps may occur. \mallocs None. */ -inline map_handle::io_result read(map_handle &self, map_handle::io_request reqs, deadline d = deadline()) noexcept +inline map_handle::io_result read(map_handle &self, map_handle::io_request reqs, + deadline d = deadline()) noexcept { return self.read(std::forward(reqs), std::forward(d)); } @@ -1038,7 +1127,8 @@ The size of each scatter-gather buffer is updated with the number of bytes of th \errors None, though the various signals and structured exception throws common to using memory maps may occur. \mallocs None. */ -inline map_handle::io_result write(map_handle &self, map_handle::io_request reqs, deadline d = deadline()) noexcept +inline map_handle::io_result write(map_handle &self, map_handle::io_request reqs, + deadline d = deadline()) noexcept { return self.write(std::forward(reqs), std::forward(d)); } @@ -1094,6 +1184,7 @@ LLFIO_V2_NAMESPACE_END #include "detail/impl/posix/map_handle.ipp" #endif #undef LLFIO_INCLUDED_BY_HEADER +#include "detail/impl/map_handle.ipp" #endif #ifdef _MSC_VER -- cgit v1.2.3