Using the new bitwise tries algorithm in QuickCppLib, implement map_handle caching. Still need to write a test for this however.

6 files changed, 455 insertions, 74 deletions

diff --git a/cmake/headers.cmake b/cmake/headers.cmake
index 88c7deb9..3ecb486f 100644
--- a/cmake/headers.cmake
+++ b/cmake/headers.cmake
@@ -32,6 +32,7 @@ set(llfio_HEADERS
   "include/llfio/v2.0/detail/impl/dynamic_thread_pool_group.ipp"
   "include/llfio/v2.0/detail/impl/fast_random_file_handle.ipp"
   "include/llfio/v2.0/detail/impl/io_multiplexer.ipp"
+  "include/llfio/v2.0/detail/impl/map_handle.ipp"
   "include/llfio/v2.0/detail/impl/path_discovery.ipp"
   "include/llfio/v2.0/detail/impl/path_view.ipp"
   "include/llfio/v2.0/detail/impl/posix/directory_handle.ipp"
diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp
index 57cfc606..809c1844 100644
--- a/include/llfio/revision.hpp
+++ b/include/llfio/revision.hpp
@@ -1,4 +1,4 @@
 // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
-#define LLFIO_PREVIOUS_COMMIT_REF    5544436fd6654fcc6cd6b3a9c3f3b9b7e0b21865
-#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-08-03 19:34:21 +00:00"
-#define LLFIO_PREVIOUS_COMMIT_UNIQUE 5544436f
+#define LLFIO_PREVIOUS_COMMIT_REF    e43ef4f6953230803a30453bbab20061b009fe05
+#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-08-20 09:36:50 +00:00"
+#define LLFIO_PREVIOUS_COMMIT_UNIQUE e43ef4f6
diff --git a/include/llfio/v2.0/detail/impl/map_handle.ipp b/include/llfio/v2.0/detail/impl/map_handle.ipp
new file mode 100644
index 00000000..b4df1724
--- /dev/null
+++ b/include/llfio/v2.0/detail/impl/map_handle.ipp
@@ -0,0 +1,247 @@
+/* A handle to a source of mapped memory
+(C) 2021 Niall Douglas <http://www.nedproductions.biz/> (10 commits)
+File Created: Aug 2021
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License in the accompanying file
+Licence.txt or at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Distributed under the Boost Software License, Version 1.0.
+    (See accompanying file Licence.txt or copy at
+          http://www.boost.org/LICENSE_1_0.txt)
+*/
+
+#include "../../map_handle.hpp"
+#include "../../utils.hpp"
+
+#include <chrono>
+#include <mutex>
+
+#include <quickcpplib/algorithm/bitwise_trie.hpp>
+
+LLFIO_V2_NAMESPACE_BEGIN
+
+namespace detail
+{
+  struct map_handle_cache_item_t
+  {
+    map_handle_cache_item_t *trie_parent{nullptr};
+    map_handle_cache_item_t *trie_child[2];
+    map_handle_cache_item_t *trie_sibling[2];
+    size_t trie_key{0};  // allocation size shifted right by map_handle_cache_page_size_shift
+    size_t page_size{0};
+    void *addr{nullptr};
+    std::chrono::steady_clock::time_point when_added{std::chrono::steady_clock::now()};
+
+    map_handle_cache_item_t(size_t key, size_t _page_size, void *_addr)
+        : trie_key(key)
+        , page_size(_page_size)
+        , addr(_addr)
+    {
+    }
+  };
+  struct map_handle_cache_base_t
+  {
+    std::mutex lock;
+    size_t trie_count{0};
+    map_handle_cache_item_t *trie_children[8 * sizeof(size_t)];
+    bool trie_nobbledir{0};
+    size_t bytes_in_cache{0};
+  };
+  static const size_t page_size_shift = [] { return QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::detail::bitscanr(utils::page_size()); }();
+  class map_handle_cache_t : protected QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie<map_handle_cache_base_t, map_handle_cache_item_t>
+  {
+    using _base = QUICKCPPLIB_NAMESPACE::algorithm::bitwise_trie::bitwise_trie<map_handle_cache_base_t, map_handle_cache_item_t>;
+    using _lock_guard = std::unique_lock<std::mutex>;
+
+  public:
+    std::atomic<unsigned> do_not_store_failed_count{0};
+
+    using _base::size;
+    void *get(size_t bytes, size_t page_size)
+    {
+      const auto _bytes = bytes >> page_size_shift;
+      _lock_guard g(lock);
+      auto it = _base::find_equal_or_larger(_bytes, 7 /* 99% close to the key after page size shift */);
+      for(; it != _base::end() && page_size != it->page_size && _bytes == it->trie_key; ++it)
+      {
+      }
+      if(it == _base::end() || page_size != it->page_size || _bytes != it->trie_key)
+      {
+        return nullptr;
+      }
+      auto *p = *it;
+      _base::erase(it);
+      _base::bytes_in_cache -= bytes;
+      assert(bytes == p->trie_key << page_size_shift);
+      g.unlock();
+      void *ret = p->addr;
+      delete p;
+      return ret;
+    }
+    void add(size_t bytes, size_t page_size, void *addr)
+    {
+      const auto _bytes = bytes >> page_size_shift;
+      auto *p = new map_handle_cache_item_t(_bytes, page_size, addr);
+      _lock_guard g(lock);
+      _base::insert(p);
+      _base::bytes_in_cache += bytes;
+    }
+    map_handle::cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than)
+    {
+      _lock_guard g(lock);
+      map_handle::cache_statistics ret;
+      if(older_than != std::chrono::steady_clock::time_point())
+      {
+        for(auto it = _base::begin(); it != _base::end();)
+        {
+          if(it->when_added < older_than)
+          {
+            auto *p = *it;
+            it = _base::erase(it);
+            const auto _bytes = p->trie_key << page_size_shift;
+            _base::bytes_in_cache -= _bytes;
+            ret.bytes_just_trimmed += _bytes;
+            ret.items_just_trimmed++;
+            delete p;
+          }
+          else
+          {
+            ++it;
+          }
+        }
+      }
+      ret.items_in_cache = _base::size();
+      ret.bytes_in_cache = _base::bytes_in_cache;
+      return ret;
+    }
+  };
+  extern inline QUICKCPPLIB_SYMBOL_EXPORT map_handle_cache_t &map_handle_cache()
+  {
+    static map_handle_cache_t v;
+    return v;
+  }
+}  // namespace detail
+
+result<map_handle> map_handle::_recycled_map(size_type bytes, section_handle::flag _flag) noexcept
+{
+  if(bytes == 0u)
+  {
+    return errc::argument_out_of_domain;
+  }
+  result<map_handle> ret(map_handle(nullptr, _flag));
+  native_handle_type &nativeh = ret.value()._v;
+  OUTCOME_TRY(auto &&pagesize, detail::pagesize_from_flags(ret.value()._flag));
+  bytes = utils::round_up_to_page_size(bytes, pagesize);
+  LLFIO_LOG_FUNCTION_CALL(&ret);
+  void *addr = detail::map_handle_cache().get(bytes, pagesize);
+  if(addr == nullptr)
+  {
+    return _new_map(bytes, _flag);
+  }
+#ifdef _WIN32
+  {
+    DWORD allocation = MEM_COMMIT, prot;
+    size_t commitsize;
+    OUTCOME_TRY(win32_map_flags(nativeh, allocation, prot, commitsize, true, _flag));
+    if(VirtualAlloc(addr, bytes, allocation, prot) == nullptr)
+    {
+      return win32_error();
+    }
+    // Windows has no way of getting the kernel to prefault maps on creation, so ...
+    if(_flag & section_handle::flag::prefault)
+    {
+      using namespace windows_nt_kernel;
+      // Start an asynchronous prefetch, so it might fault the whole lot in at once
+      buffer_type b{static_cast<byte *>(addr), bytes};
+      (void) prefetch(span<buffer_type>(&b, 1));
+      volatile auto *a = static_cast<volatile char *>(addr);
+      for(size_t n = 0; n < bytes; n += pagesize)
+      {
+        a[n];
+      }
+    }
+  }
+#else
+  int flags = MAP_FIXED;
+#ifdef MAP_POPULATE
+  if(_flag & section_handle::flag::prefault)
+  {
+    flags |= MAP_POPULATE;
+  }
+#endif
+#ifdef MAP_PREFAULT_READ
+  if(_flag & section_handle::flag::prefault)
+    flags |= MAP_PREFAULT_READ;
+#endif
+  OUTCOME_TRY(do_mmap(nativeh, addr, flags, nullptr, pagesize, bytes, 0, _flag));
+#endif
+  ret.value()._addr = static_cast<byte *>(addr);
+  ret.value()._reservation = bytes;
+  ret.value()._length = bytes;
+  ret.value()._pagesize = pagesize;
+  nativeh._init = -2;  // otherwise appears closed
+  nativeh.behaviour |= native_handle_type::disposition::allocation;
+  return ret;
+}
+
+bool map_handle::_recycle_map() noexcept
+{
+  try
+  {
+    LLFIO_LOG_FUNCTION_CALL(this);
+    auto &c = detail::map_handle_cache();
+#ifdef _WIN32
+    if(!win32_release_nonfile_allocations(_addr, _length, MEM_DECOMMIT))
+    {
+      return false;
+    }
+#else
+    if(memory_accounting() == memory_accounting_kind::commit_charge)
+    {
+      if(!do_mmap(_v, _addr, MAP_FIXED, nullptr, _pagesize, _length, 0, section_handle::flag::none | section_handle::flag::nocommit))
+      {
+        return false;
+      }
+    }
+    else
+    {
+#ifdef __linux__
+      if(c.do_not_store_failed_count.load(std::memory_order_relaxed) < 10)
+      {
+        auto r = do_not_store({_addr, _length});
+        if(!r)
+        {
+          c.do_not_store_failed_count.fetch_add(1, std::memory_order_relaxed);
+        }
+      }
+#endif
+    }
+#endif
+    c.add(_length, _pagesize, _addr);
+    return true;  // cached
+  }
+  catch(...)
+  {
+    return false;
+  }
+}
+
+map_handle::cache_statistics map_handle::trim_cache(std::chrono::steady_clock::time_point older_than) noexcept
+{
+  return detail::map_handle_cache().trim_cache(older_than);
+}
+
+
+LLFIO_V2_NAMESPACE_END
diff --git a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp
index 397576a8..f77954ca 100644
--- a/include/llfio/v2.0/detail/impl/posix/map_handle.ipp
+++ b/include/llfio/v2.0/detail/impl/posix/map_handle.ipp
@@ -1,5 +1,5 @@
 /* A handle to a source of mapped memory
-(C) 2017-2020 Niall Douglas <http://www.nedproductions.biz/> (10 commits)
+(C) 2017-2021 Niall Douglas <http://www.nedproductions.biz/> (10 commits)
 File Created: Apr 2017
 
 
@@ -156,8 +156,10 @@ map_handle::~map_handle()
     auto ret = map_handle::close();
     if(ret.has_error())
     {
-      LLFIO_LOG_FATAL(_v.fd, "map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the 64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to "
-                             "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out.");
+      LLFIO_LOG_FATAL(_v.fd,
+                      "map_handle::~map_handle() close failed. Cause is typically other code modifying mapped regions. If on Linux, you may have exceeded the "
+                      "64k VMA process limit, set the LLFIO_DEBUG_LINUX_MUNMAP macro at the top of posix/map_handle.ipp to cause dumping of VMAs to "
+                      "/tmp/llfio_unmap_debug_smaps.txt, and combine with strace to figure it out.");
       abort();
     }
   }
@@ -173,27 +175,31 @@ result<void> map_handle::close() noexcept
       OUTCOME_TRYV(map_handle::barrier(barrier_kind::wait_all));
     }
     // printf("%d munmap %p-%p\n", getpid(), _addr, _addr+_reservation);
-    if(-1 == ::munmap(_addr, _reservation))
+    if(_section != nullptr || !_recycle_map())
     {
-#ifdef LLFIO_DEBUG_LINUX_MUNMAP
-      int olderrno = errno;
-      ssize_t bytesread;
-      // Refresh the /proc file
-      (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_END);
-      (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_SET);
-      char buffer[4096];
-      (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, sprintf(buffer, "\n---\nCause of munmap failure by process %d: %d (%s)\n\n", getpid(), olderrno, strerror(olderrno)));
-      do
+      if(-1 == ::munmap(_addr, _reservation))
       {
-        bytesread = ::read(llfio_linux_munmap_debug.smaps_fd, buffer, sizeof(buffer));
-        if(bytesread > 0)
+#ifdef LLFIO_DEBUG_LINUX_MUNMAP
+        int olderrno = errno;
+        ssize_t bytesread;
+        // Refresh the /proc file
+        (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_END);
+        (void) ::lseek(llfio_linux_munmap_debug.smaps_fd, 0, SEEK_SET);
+        char buffer[4096];
+        (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer,
+                       sprintf(buffer, "\n---\nCause of munmap failure by process %d: %d (%s)\n\n", getpid(), olderrno, strerror(olderrno)));
+        do
         {
-          (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, bytesread);
-        }
-      } while(bytesread > 0);
-      errno = olderrno;
+          bytesread = ::read(llfio_linux_munmap_debug.smaps_fd, buffer, sizeof(buffer));
+          if(bytesread > 0)
+          {
+            (void) ::write(llfio_linux_munmap_debug.dumpfile_fd, buffer, bytesread);
+          }
+        } while(bytesread > 0);
+        errno = olderrno;
 #endif
-      return posix_error();
+        return posix_error();
+      }
     }
   }
   // We don't want ~handle() to close our borrowed handle
@@ -213,7 +219,8 @@ native_handle_type map_handle::release() noexcept
   return {};
 }
 
-map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_barrier(map_handle::io_request<map_handle::const_buffers_type> reqs, barrier_kind kind, deadline d) noexcept
+map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_barrier(map_handle::io_request<map_handle::const_buffers_type> reqs, barrier_kind kind,
+                                                                              deadline d) noexcept
 {
   LLFIO_LOG_FUNCTION_CALL(this);
   byte *addr = _addr + reqs.offset;
@@ -255,8 +262,38 @@ map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_barrier(ma
   return {reqs.buffers};
 }
 
+#ifdef __linux__
+LLFIO_HEADERS_ONLY_MEMFUNC_SPEC map_handle::memory_accounting_kind map_handle::memory_accounting() noexcept
+{
+  static memory_accounting_kind v{memory_accounting_kind::unknown};
+  if(v != memory_accounting_kind::unknown)
+  {
+    return v;
+  }
+  int fd = ::open("/proc/sys/vm/overcommit_memory", O_RDONLY);
+  if(fd != -1)
+  {
+    char buffer[8];
+    if(::read(fd, buffer, 8) > 0)
+    {
+      if(buffer[0] == '2')
+      {
+        v = memory_accounting_kind::commit_charge;
+      }
+      else
+      {
+        v = memory_accounting_kind::over_commit;
+      }
+    }
+    ::close(fd);
+  }
+  return v;
+}
+#endif
+
 
-static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, int extra_flags, section_handle *section, map_handle::size_type pagesize, map_handle::size_type &bytes, map_handle::extent_type offset, section_handle::flag _flag) noexcept
+static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr, int extra_flags, section_handle *section, map_handle::size_type pagesize,
+                                     map_handle::size_type &bytes, map_handle::extent_type offset, section_handle::flag _flag) noexcept
 {
   bool have_backing = (section != nullptr);
   int prot = 0, flags = have_backing ? MAP_SHARED : (MAP_PRIVATE | MAP_ANONYMOUS);
@@ -387,9 +424,8 @@ static inline result<void *> do_mmap(native_handle_type &nativeh, void *ataddr,
   return addr;
 }
 
-result<map_handle> map_handle::map(size_type bytes, bool /*unused*/, section_handle::flag _flag) noexcept
+result<map_handle> map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept
 {
-  // TODO: Keep a cache of MADV_FREE pages deallocated
   if(bytes == 0u)
   {
     return errc::argument_out_of_domain;
@@ -495,7 +531,7 @@ result<map_handle::size_type> map_handle::truncate(size_type newsize, bool permi
     _reservation = _newsize;
     _length = (length - _offset < newsize) ? (length - _offset) : newsize;  // length of backing, not reservation
     return newsize;
-#else                  // generic POSIX, inefficient
+#else  // generic POSIX, inefficient
     byte *addrafter = _addr + _reservation;
     size_type bytes = newsize - _reservation;
     extent_type offset = _offset + _reservation;
diff --git a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp
index f1996484..4a175f89 100644
--- a/include/llfio/v2.0/detail/impl/windows/map_handle.ipp
+++ b/include/llfio/v2.0/detail/impl/windows/map_handle.ipp
@@ -1,5 +1,5 @@
 /* A handle to a source of mapped memory
-(C) 2016-2020 Niall Douglas <http://www.nedproductions.biz/> (17 commits)
+(C) 2016-2021 Niall Douglas <http://www.nedproductions.biz/> (17 commits)
 File Created: August 2016
 
 
@@ -554,7 +554,10 @@ result<void> map_handle::close() noexcept
     }
     else
     {
-      OUTCOME_TRYV(win32_release_nonfile_allocations(_addr, _reservation, MEM_RELEASE));
+      if(!_recycle_map())
+      {
+        OUTCOME_TRYV(win32_release_nonfile_allocations(_addr, _reservation, MEM_RELEASE));
+      }
     }
   }
   // We don't want ~handle() to close our borrowed handle
@@ -619,9 +622,12 @@ map_handle::io_result<map_handle::const_buffers_type> map_handle::_do_barrier(ma
 }
 
 
-result<map_handle> map_handle::map(size_type bytes, bool /*unused*/, section_handle::flag _flag) noexcept
+result<map_handle> map_handle::_new_map(size_type bytes, section_handle::flag _flag) noexcept
 {
-  // TODO: Keep a cache of DiscardVirtualMemory()/MEM_RESET pages deallocated
+  if(bytes == 0u)
+  {
+    return errc::argument_out_of_domain;
+  }
   result<map_handle> ret(map_handle(nullptr, _flag));
   native_handle_type &nativeh = ret.value()._v;
   DWORD allocation = MEM_RESERVE | MEM_COMMIT, prot;
diff --git a/include/llfio/v2.0/map_handle.hpp b/include/llfio/v2.0/map_handle.hpp
index 69241720..ba6b83ef 100644
--- a/include/llfio/v2.0/map_handle.hpp
+++ b/include/llfio/v2.0/map_handle.hpp
@@ -1,5 +1,5 @@
 /* A handle to a source of mapped memory
-(C) 2016-2018 Niall Douglas <http://www.nedproductions.biz/> (14 commits)
+(C) 2016-2021 Niall Douglas <http://www.nedproductions.biz/> (14 commits)
 File Created: August 2016
 
 
@@ -55,28 +55,32 @@ public:
   using size_type = handle::size_type;
 
   //! The behaviour of the memory section
-  QUICKCPPLIB_BITFIELD_BEGIN(flag){none = 0U,           //!< No flags
-                                   read = 1U << 0U,     //!< Memory views can be read
-                                   write = 1U << 1U,    //!< Memory views can be written
-                                   cow = 1U << 2U,      //!< Memory views can be copy on written
-                                   execute = 1U << 3U,  //!< Memory views can execute code
-
-                                   nocommit = 1U << 8U,     //!< Don't allocate space for this memory in the system immediately
-                                   prefault = 1U << 9U,     //!< Prefault, as if by reading every page, any views of memory upon creation.
-                                   executable = 1U << 10U,  //!< The backing storage is in fact an executable program binary.
-                                   singleton = 1U << 11U,   //!< A single instance of this section is to be shared by all processes using the same backing file.
-
-                                   barrier_on_close = 1U << 16U,   //!< Maps of this section, if writable, issue a `barrier()` when destructed blocking until data (not metadata) reaches physical storage.
-                                   nvram = 1U << 17U,              //!< This section is of non-volatile RAM.
-                                   write_via_syscall = 1U << 18U,  //!< For file backed maps, `map_handle::write()` is implemented as a `write()` syscall to the file descriptor. This causes the map to be mapped read-only.
-
-                                   page_sizes_1 = 1U << 24U,  //!< Use `utils::page_sizes()[1]` sized pages, or fail.
-                                   page_sizes_2 = 2U << 24U,  //!< Use `utils::page_sizes()[2]` sized pages, or fail.
-                                   page_sizes_3 = 3U << 24U,  //!< Use `utils::page_sizes()[3]` sized pages, or fail.
-
-                                   // NOTE: IF UPDATING THIS UPDATE THE std::ostream PRINTER BELOW!!!
-
-                                   readwrite = (read | write)};
+  QUICKCPPLIB_BITFIELD_BEGIN(flag){
+  none = 0U,           //!< No flags
+  read = 1U << 0U,     //!< Memory views can be read
+  write = 1U << 1U,    //!< Memory views can be written
+  cow = 1U << 2U,      //!< Memory views can be copy on written
+  execute = 1U << 3U,  //!< Memory views can execute code
+
+  nocommit = 1U << 8U,     //!< Don't allocate space for this memory in the system immediately
+  prefault = 1U << 9U,     //!< Prefault, as if by reading every page, any views of memory upon creation.
+  executable = 1U << 10U,  //!< The backing storage is in fact an executable program binary.
+  singleton = 1U << 11U,   //!< A single instance of this section is to be shared by all processes using the same backing file.
+
+  barrier_on_close =
+  1U << 16U,          //!< Maps of this section, if writable, issue a `barrier()` when destructed blocking until data (not metadata) reaches physical storage.
+  nvram = 1U << 17U,  //!< This section is of non-volatile RAM.
+  write_via_syscall =
+  1U
+  << 18U,  //!< For file backed maps, `map_handle::write()` is implemented as a `write()` syscall to the file descriptor. This causes the map to be mapped read-only.
+
+  page_sizes_1 = 1U << 24U,  //!< Use `utils::page_sizes()[1]` sized pages, or fail.
+  page_sizes_2 = 2U << 24U,  //!< Use `utils::page_sizes()[2]` sized pages, or fail.
+  page_sizes_3 = 3U << 24U,  //!< Use `utils::page_sizes()[3]` sized pages, or fail.
+
+  // NOTE: IF UPDATING THIS UPDATE THE std::ostream PRINTER BELOW!!!
+
+  readwrite = (read | write)};
   QUICKCPPLIB_BITFIELD_END(flag);
 
 protected:
@@ -154,7 +158,10 @@ public:
   \errors Any of the values POSIX dup(), open() or NtCreateSection() can return.
   */
   LLFIO_MAKE_FREE_FUNCTION
-  static result<section_handle> section(file_handle &backing, extent_type bytes = 0) noexcept { return section(backing, bytes, backing.is_writable() ? (flag::readwrite) : (flag::read)); }
+  static result<section_handle> section(file_handle &backing, extent_type bytes = 0) noexcept
+  {
+    return section(backing, bytes, backing.is_writable() ? (flag::readwrite) : (flag::read));
+  }
   /*! \brief Create a memory section backed by an anonymous, managed file.
   \param bytes The initial size of this section. Cannot be zero.
   \param dirh Where to create the anonymous, managed file.
@@ -163,7 +170,9 @@ public:
   \errors Any of the values POSIX dup(), open() or NtCreateSection() can return.
   */
   LLFIO_MAKE_FREE_FUNCTION
-  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<section_handle> section(extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), flag _flag = flag::read | flag::write) noexcept;
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<section_handle> section(extent_type bytes,
+                                                                        const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(),
+                                                                        flag _flag = flag::read | flag::write) noexcept;
 
   //! Returns the memory section's flags
   flag section_flags() const noexcept { return _flag; }
@@ -454,7 +463,8 @@ protected:
     if(section != nullptr)
     {
       // Apart from read/write/cow/execute, the section's flags override the map's flags
-      _flag |= (section->section_flags() & ~(section_handle::flag::read | section_handle::flag::write | section_handle::flag::cow | section_handle::flag::execute));
+      _flag |=
+      (section->section_flags() & ~(section_handle::flag::read | section_handle::flag::write | section_handle::flag::cow | section_handle::flag::execute));
     }
   }
 
@@ -463,7 +473,8 @@ public:
   constexpr map_handle() {}  // NOLINT
   LLFIO_HEADERS_ONLY_VIRTUAL_SPEC ~map_handle() override;
   //! Construct an instance managing pages at `addr`, `length`, `pagesize` and `flags`
-  explicit map_handle(byte *addr, size_type length, size_type pagesize, section_handle::flag flags, section_handle *section = nullptr, extent_type offset = 0) noexcept
+  explicit map_handle(byte *addr, size_type length, size_type pagesize, section_handle::flag flags, section_handle *section = nullptr,
+                      extent_type offset = 0) noexcept
       : _section(section)
       , _addr(addr)
       , _offset(offset)
@@ -526,10 +537,18 @@ public:
 
 protected:
   LLFIO_HEADERS_ONLY_VIRTUAL_SPEC size_t _do_max_buffers() const noexcept override { return 0; }
-  LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> _do_barrier(io_request<const_buffers_type> reqs = io_request<const_buffers_type>(), barrier_kind kind = barrier_kind::nowait_data_only, deadline d = deadline()) noexcept override;
+  LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> _do_barrier(io_request<const_buffers_type> reqs = io_request<const_buffers_type>(),
+                                                                            barrier_kind kind = barrier_kind::nowait_data_only,
+                                                                            deadline d = deadline()) noexcept override;
   LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<buffers_type> _do_read(io_request<buffers_type> reqs, deadline d = deadline()) noexcept override;
   LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> _do_write(io_request<const_buffers_type> reqs, deadline d = deadline()) noexcept override;
 
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<map_handle> _new_map(size_type bytes, section_handle::flag _flag) noexcept;
+
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<map_handle> _recycled_map(size_type bytes, section_handle::flag _flag) noexcept;
+
+  LLFIO_HEADERS_ONLY_MEMFUNC_SPEC bool _recycle_map() noexcept;
+
 public:
   /*! Map unused memory into view, creating new memory if insufficient unused memory is available
   (i.e. add the returned memory to the process' commit charge, unless `flag::nocommit`
@@ -538,7 +557,9 @@ public:
 
   \param bytes How many bytes to map. Typically will be rounded up to a multiple of the page size
   (see `page_size()`).
-  \param zeroed Set to true if only all bits zeroed memory is wanted.
+  \param zeroed Set to true if only all bits zeroed memory is wanted. If this is true, a syscall
+  is always performed as the kernel probably has zeroed pages ready to go, whereas if false, the
+  request may be satisfied from a local cache instead. The default is false.
   \param _flag The permissions with which to map the view.
 
   \note On Microsoft Windows this constructor uses the faster `VirtualAlloc()` which creates less
@@ -547,10 +568,35 @@ public:
   the other constructor. This makes available all those very useful VM tricks Windows can do with
   section mapped memory which `VirtualAlloc()` memory cannot do.
 
+  When this kind of map handle is closed, it is added to an internal cache so new map
+  handle creations of this kind with `zeroed = false` are very quick and avoid a syscall. The
+  internal cache may return a map slightly bigger than requested. If you wish to always invoke
+  the syscall, specify `zeroed = true`.
+
+  When maps are added to the internal cache, on all systems except Linux the memory is decommitted
+  first. This reduces commit charge appropriately, thus only virtual address space remains
+  consumed. On Linux, if `memory_accounting()` is `memory_accounting_kind::commit_charge`, we also
+  decommit, however be aware that this can increase the average VMA use count in the Linux kernel, and most
+  Linux kernels are configured with a very low per-process limit of 64k VMAs (this is easy to raise
+  using `sysctl -w vm.max_map_count=262144`). Otherwise on Linux to avoid increasing VMA count
+  we instead mark closed maps as `LazyFree`, which means that their contents can be arbitrarily
+  disposed of by the Linux kernel as needed, but also allows Linux to coalesce VMAs so the very
+  low per-process limit is less likely to be exceeded. If the `LazyFree` syscall is not implemented on this
+  Linux, we do nothing.
+
+  \warning The cache does not self-trim on its own, you MUST call `trim_cache()` to trim allocations of
+  virtual address (these don't count towards process commit charge, but they do consume address space
+  and precious VMAs in the Linux kernel). Only on 32 bit processes where virtual address space is limited, or on
+  Linux where VMAs allocated is considered by the Linux OOM killer, will you need to probably
+  care much about regular cache trimming.
+
   \errors Any of the values POSIX `mmap()` or `VirtualAlloc()` can return.
   */
   LLFIO_MAKE_FREE_FUNCTION
-  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<map_handle> map(size_type bytes, bool zeroed = false, section_handle::flag _flag = section_handle::flag::readwrite) noexcept;
+  static inline result<map_handle> map(size_type bytes, bool zeroed = false, section_handle::flag _flag = section_handle::flag::readwrite) noexcept
+  {
+    return (zeroed || (_flag & section_handle::flag::nocommit)) ? _new_map(bytes, _flag) : _recycled_map(bytes, _flag);
+  }
 
   /*! Reserve address space within which individual pages can later be committed. Reserved address
   space is NOT added to the process' commit charge.
@@ -566,7 +612,7 @@ public:
   \errors Any of the values POSIX `mmap()` or `VirtualAlloc()` can return.
   */
   LLFIO_MAKE_FREE_FUNCTION
-  static inline result<map_handle> reserve(size_type bytes) noexcept { return map(bytes, false, section_handle::flag::none | section_handle::flag::nocommit); }
+  static inline result<map_handle> reserve(size_type bytes) noexcept { return _new_map(bytes, section_handle::flag::none | section_handle::flag::nocommit); }
 
   /*! Create a memory mapped view of a backing storage, optionally reserving additional address
   space for later growth.
@@ -585,7 +631,41 @@ public:
   \errors Any of the values POSIX `mmap()` or `NtMapViewOfSection()` can return.
   */
   LLFIO_MAKE_FREE_FUNCTION
-  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<map_handle> map(section_handle &section, size_type bytes = 0, extent_type offset = 0, section_handle::flag _flag = section_handle::flag::readwrite) noexcept;
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<map_handle> map(section_handle &section, size_type bytes = 0, extent_type offset = 0,
+                                                                section_handle::flag _flag = section_handle::flag::readwrite) noexcept;
+
+  //! The kind of memory accounting this system uses
+  enum class memory_accounting_kind
+  {
+    unknown,
+    /*! This system will not permit more than physical RAM and your swap files to be committed.
+    On every OS except for Linux, this is always the case.
+    */
+    commit_charge,
+    /*! This system will permit more memory to be committed than physical RAM and your swap
+    files, and will terminate your process without warning at some unknown point
+    if you write into enough of the pages committed. This is typically the default on Linux,
+    but it can be changed at runtime.
+    */
+    over_commit
+  };
+#ifdef __linux__
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC memory_accounting_kind memory_accounting() noexcept;
+#else
+  static memory_accounting_kind memory_accounting() noexcept { return memory_accounting_kind::commit_charge; }
+#endif
+
+  //! Statistics about the map handle cache
+  struct cache_statistics
+  {
+    size_t items_in_cache{0};
+    size_t bytes_in_cache{0};
+    size_t items_just_trimmed{0};
+    size_t bytes_just_trimmed{0};
+  };
+  /*! Get statistics about the map handle cache, optionally trimming the least recently used maps.
+   */
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC cache_statistics trim_cache(std::chrono::steady_clock::time_point older_than = {}) noexcept;
 
   //! The memory section this handle is using
   section_handle *section() const noexcept { return _section; }
@@ -718,7 +798,8 @@ public:
   */
   LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<buffer_type> do_not_store(buffer_type region) noexcept;
 
-  //! Ask the system to begin to asynchronously prefetch the span of memory regions given, returning the regions actually prefetched. Note that on Windows 7 or earlier the system call to implement this was not available, and so you will see an empty span returned.
+  //! Ask the system to begin to asynchronously prefetch the span of memory regions given, returning the regions actually prefetched. Note that on Windows 7 or
+  //! earlier the system call to implement this was not available, and so you will see an empty span returned.
   static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<span<buffer_type>> prefetch(span<buffer_type> regions) noexcept;
   //! \overload
   static result<buffer_type> prefetch(buffer_type region) noexcept
@@ -898,7 +979,8 @@ inline void swap(section_handle &self, section_handle &o) noexcept
 */
 inline result<section_handle> section(file_handle &backing, section_handle::extent_type maximum_size, section_handle::flag _flag) noexcept
 {
-  return section_handle::section(std::forward<decltype(backing)>(backing), std::forward<decltype(maximum_size)>(maximum_size), std::forward<decltype(_flag)>(_flag));
+  return section_handle::section(std::forward<decltype(backing)>(backing), std::forward<decltype(maximum_size)>(maximum_size),
+                                 std::forward<decltype(_flag)>(_flag));
 }
 /*! \brief Create a memory section backed by a file.
 \param backing The handle to use as backing storage.
@@ -919,7 +1001,8 @@ inline result<section_handle> section(file_handle &backing, section_handle::exte
 
 \errors Any of the values POSIX dup(), open() or NtCreateSection() can return.
 */
-inline result<section_handle> section(section_handle::extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(), section_handle::flag _flag = section_handle::flag::read | section_handle::flag::write) noexcept
+inline result<section_handle> section(section_handle::extent_type bytes, const path_handle &dirh = path_discovery::storage_backed_temporary_files_directory(),
+                                      section_handle::flag _flag = section_handle::flag::read | section_handle::flag::write) noexcept
 {
   return section_handle::section(std::forward<decltype(bytes)>(bytes), std::forward<decltype(dirh)>(dirh), std::forward<decltype(_flag)>(_flag));
 }
@@ -952,7 +1035,8 @@ inline result<void> close(map_handle &self) noexcept
 /*! Create new memory and map it into view.
 \param bytes How many bytes to create and map. Typically will be rounded up to a multiple of the page size (see `page_size()`) on POSIX, 64Kb on Windows.
 \param zeroed Set to true if only all bits zeroed memory is wanted.
-\param _flag The permissions with which to map the view. `flag::none` can be useful for reserving virtual address space without committing system resources, use commit() to later change availability of memory.
+\param _flag The permissions with which to map the view. `flag::none` can be useful for reserving virtual address space without committing system resources, use
+commit() to later change availability of memory.
 
 \note On Microsoft Windows this constructor uses the faster VirtualAlloc() which creates less versatile page backed memory. If you want anonymous memory
 allocated from a paging file backed section instead, create a page file backed section and then a mapped view from that using
@@ -968,14 +1052,18 @@ inline result<map_handle> map(map_handle::size_type bytes, bool zeroed = false,
 /*! Create a memory mapped view of a backing storage, optionally reserving additional address space for later growth.
 \param section A memory section handle specifying the backing storage to use.
 \param bytes How many bytes to reserve (0 = the size of the section). Rounded up to nearest 64Kb on Windows.
-\param offset The offset into the backing storage to map from. Typically needs to be at least a multiple of the page size (see `page_size()`), on Windows it needs to be a multiple of the kernel memory allocation granularity (typically 64Kb).
-\param _flag The permissions with which to map the view which are constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use commit() to later change availability of memory.
+\param offset The offset into the backing storage to map from. Typically needs to be at least a multiple of the page size (see `page_size()`), on Windows it
+needs to be a multiple of the kernel memory allocation granularity (typically 64Kb). \param _flag The permissions with which to map the view which are
+constrained by the permissions of the memory section. `flag::none` can be useful for reserving virtual address space without committing system resources, use
+commit() to later change availability of memory.
 
 \errors Any of the values POSIX mmap() or NtMapViewOfSection() can return.
 */
-inline result<map_handle> map(section_handle &section, map_handle::size_type bytes = 0, map_handle::extent_type offset = 0, section_handle::flag _flag = section_handle::flag::readwrite) noexcept
+inline result<map_handle> map(section_handle &section, map_handle::size_type bytes = 0, map_handle::extent_type offset = 0,
+                              section_handle::flag _flag = section_handle::flag::readwrite) noexcept
 {
-  return map_handle::map(std::forward<decltype(section)>(section), std::forward<decltype(bytes)>(bytes), std::forward<decltype(offset)>(offset), std::forward<decltype(_flag)>(_flag));
+  return map_handle::map(std::forward<decltype(section)>(section), std::forward<decltype(bytes)>(bytes), std::forward<decltype(offset)>(offset),
+                         std::forward<decltype(_flag)>(_flag));
 }
 //! The size of the memory map. This is the accessible size, NOT the reservation size.
 inline map_handle::size_type length(const map_handle &self) noexcept
@@ -1024,7 +1112,8 @@ The size of each scatter-gather buffer is updated with the number of bytes of th
 \errors None, though the various signals and structured exception throws common to using memory maps may occur.
 \mallocs None.
 */
-inline map_handle::io_result<map_handle::buffers_type> read(map_handle &self, map_handle::io_request<map_handle::buffers_type> reqs, deadline d = deadline()) noexcept
+inline map_handle::io_result<map_handle::buffers_type> read(map_handle &self, map_handle::io_request<map_handle::buffers_type> reqs,
+                                                            deadline d = deadline()) noexcept
 {
   return self.read(std::forward<decltype(reqs)>(reqs), std::forward<decltype(d)>(d));
 }
@@ -1038,7 +1127,8 @@ The size of each scatter-gather buffer is updated with the number of bytes of th
 \errors None, though the various signals and structured exception throws common to using memory maps may occur.
 \mallocs None.
 */
-inline map_handle::io_result<map_handle::const_buffers_type> write(map_handle &self, map_handle::io_request<map_handle::const_buffers_type> reqs, deadline d = deadline()) noexcept
+inline map_handle::io_result<map_handle::const_buffers_type> write(map_handle &self, map_handle::io_request<map_handle::const_buffers_type> reqs,
+                                                                   deadline d = deadline()) noexcept
 {
   return self.write(std::forward<decltype(reqs)>(reqs), std::forward<decltype(d)>(d));
 }
@@ -1094,6 +1184,7 @@ LLFIO_V2_NAMESPACE_END
 #include "detail/impl/posix/map_handle.ipp"
 #endif
 #undef LLFIO_INCLUDED_BY_HEADER
+#include "detail/impl/map_handle.ipp"
 #endif
 
 #ifdef _MSC_VER