Add statfs_t::f_iosinprogress and statfs_t::f_ioswaittime which

can be used to monitor how congested the underlying hardware
device for a handle is.

9 files changed, 634 insertions, 171 deletions

diff --git a/cmake/tests.cmake b/cmake/tests.cmake
index b216ccba..47e3ba9e 100644
--- a/cmake/tests.cmake
+++ b/cmake/tests.cmake
@@ -28,6 +28,7 @@ set(llfio_TESTS
   "test/tests/section_handle_create_close/kernel_section_handle.cpp.hpp"
   "test/tests/section_handle_create_close/runner.cpp"
   "test/tests/shared_fs_mutex.cpp"
+  "test/tests/statfs.cpp"
   "test/tests/symlink_handle_create_close/kernel_symlink_handle.cpp.hpp"
   "test/tests/symlink_handle_create_close/runner.cpp"
   "test/tests/traverse.cpp"
diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp
index 4f6234d6..9e73339c 100644
--- a/include/llfio/revision.hpp
+++ b/include/llfio/revision.hpp
@@ -1,4 +1,4 @@
 // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
-#define LLFIO_PREVIOUS_COMMIT_REF    545a722a055dcc38e7f80520a7a34008bfa9a86f
-#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-03-15 10:40:32 +00:00"
-#define LLFIO_PREVIOUS_COMMIT_UNIQUE 545a722a
+#define LLFIO_PREVIOUS_COMMIT_REF    540cff7aa6d51cdad25c50857a4c3f523368cd33
+#define LLFIO_PREVIOUS_COMMIT_DATE   "2020-12-17 13:41:30 +00:00"
+#define LLFIO_PREVIOUS_COMMIT_UNIQUE 540cff7a
diff --git a/include/llfio/v2.0/detail/impl/posix/statfs.ipp b/include/llfio/v2.0/detail/impl/posix/statfs.ipp
index a853a418..8b2ebcf3 100644
--- a/include/llfio/v2.0/detail/impl/posix/statfs.ipp
+++ b/include/llfio/v2.0/detail/impl/posix/statfs.ipp
@@ -25,10 +25,15 @@ Distributed under the Boost Software License, Version 1.0.
 #include "../../../handle.hpp"
 #include "../../../statfs.hpp"
 
+#include <chrono>
+#include <mutex>
+#include <vector>
+
 #include <sys/mount.h>
 #ifdef __linux__
 #include <mntent.h>
 #include <sys/statfs.h>
+#include <sys/sysmacros.h>
 #endif
 
 LLFIO_V2_NAMESPACE_BEGIN
@@ -37,186 +42,189 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<size_t> statfs_t::fill(const handle &h, s
 {
   size_t ret = 0;
 #ifdef __linux__
-  struct statfs64 s
-  {
-  };
-  memset(&s, 0, sizeof(s));
-  if(-1 == fstatfs64(h.native_handle().fd, &s))
-  {
-    return posix_error();
-  }
-  if(!!(wanted & want::bsize))
+  if(!!(wanted & ~(want::iosinprogress | want::ioswaittime)))
   {
-    f_bsize = s.f_bsize;
-    ++ret;
-  }
-  if(!!(wanted & want::iosize))
-  {
-    f_iosize = s.f_frsize;
-    ++ret;
-  }
-  if(!!(wanted & want::blocks))
-  {
-    f_blocks = s.f_blocks;
-    ++ret;
-  }
-  if(!!(wanted & want::bfree))
-  {
-    f_bfree = s.f_bfree;
-    ++ret;
-  }
-  if(!!(wanted & want::bavail))
-  {
-    f_bavail = s.f_bavail;
-    ++ret;
-  }
-  if(!!(wanted & want::files))
-  {
-    f_files = s.f_files;
-    ++ret;
-  }
-  if(!!(wanted & want::ffree))
-  {
-    f_ffree = s.f_ffree;
-    ++ret;
-  }
-  if(!!(wanted & want::namemax))
-  {
-    f_namemax = s.f_namelen;
-    ++ret;
-  }
-  //            if(!!(wanted&&want::owner))       { f_owner      =s.f_owner;  ++ret; }
-  if(!!(wanted & want::fsid))
-  {
-    f_fsid[0] = static_cast<unsigned>(s.f_fsid.__val[0]);
-    f_fsid[1] = static_cast<unsigned>(s.f_fsid.__val[1]);
-    ++ret;
-  }
-  if(!!(wanted & want::flags) || !!(wanted & want::fstypename) || !!(wanted & want::mntfromname) || !!(wanted & want::mntonname))
-  {
-    try
+    struct statfs64 s
     {
-      struct mountentry
-      {
-        std::string mnt_fsname, mnt_dir, mnt_type, mnt_opts;
-        mountentry(const char *a, const char *b, const char *c, const char *d)
-            : mnt_fsname(a)
-            , mnt_dir(b)
-            , mnt_type(c)
-            , mnt_opts(d)
-        {
-        }
-      };
-      std::vector<std::pair<mountentry, struct statfs64>> mountentries;
+    };
+    memset(&s, 0, sizeof(s));
+    if(-1 == fstatfs64(h.native_handle().fd, &s))
+    {
+      return posix_error();
+    }
+    if(!!(wanted & want::bsize))
+    {
+      f_bsize = s.f_bsize;
+      ++ret;
+    }
+    if(!!(wanted & want::iosize))
+    {
+      f_iosize = s.f_frsize;
+      ++ret;
+    }
+    if(!!(wanted & want::blocks))
+    {
+      f_blocks = s.f_blocks;
+      ++ret;
+    }
+    if(!!(wanted & want::bfree))
+    {
+      f_bfree = s.f_bfree;
+      ++ret;
+    }
+    if(!!(wanted & want::bavail))
+    {
+      f_bavail = s.f_bavail;
+      ++ret;
+    }
+    if(!!(wanted & want::files))
+    {
+      f_files = s.f_files;
+      ++ret;
+    }
+    if(!!(wanted & want::ffree))
+    {
+      f_ffree = s.f_ffree;
+      ++ret;
+    }
+    if(!!(wanted & want::namemax))
+    {
+      f_namemax = s.f_namelen;
+      ++ret;
+    }
+    //            if(!!(wanted&&want::owner))       { f_owner      =s.f_owner;  ++ret; }
+    if(!!(wanted & want::fsid))
+    {
+      f_fsid[0] = static_cast<unsigned>(s.f_fsid.__val[0]);
+      f_fsid[1] = static_cast<unsigned>(s.f_fsid.__val[1]);
+      ++ret;
+    }
+    if(!!(wanted & want::flags) || !!(wanted & want::fstypename) || !!(wanted & want::mntfromname) || !!(wanted & want::mntonname))
+    {
+      try
       {
-        // Need to parse mount options on Linux
-        FILE *mtab = setmntent("/etc/mtab", "r");
-        if(mtab == nullptr)
-        {
-          mtab = setmntent("/proc/mounts", "r");
-        }
-        if(mtab == nullptr)
-        {
-          return posix_error();
-        }
-        auto unmtab = make_scope_exit([mtab]() noexcept { endmntent(mtab); });
-        struct mntent m
+        struct mountentry
         {
+          std::string mnt_fsname, mnt_dir, mnt_type, mnt_opts;
+          mountentry(const char *a, const char *b, const char *c, const char *d)
+              : mnt_fsname(a)
+              , mnt_dir(b)
+              , mnt_type(c)
+              , mnt_opts(d)
+          {
+          }
         };
-        char buffer[32768];
-        while(getmntent_r(mtab, &m, buffer, sizeof(buffer)) != nullptr)
+        std::vector<std::pair<mountentry, struct statfs64>> mountentries;
         {
-          struct statfs64 temp
+          // Need to parse mount options on Linux
+          FILE *mtab = setmntent("/etc/mtab", "r");
+          if(mtab == nullptr)
+          {
+            mtab = setmntent("/proc/mounts", "r");
+          }
+          if(mtab == nullptr)
+          {
+            return posix_error();
+          }
+          auto unmtab = make_scope_exit([mtab]() noexcept { endmntent(mtab); });
+          struct mntent m
           {
           };
-          memset(&temp, 0, sizeof(temp));
-          // std::cout << m.mnt_fsname << "," << m.mnt_dir << "," << m.mnt_type << "," << m.mnt_opts << std::endl;
-          if(0 == statfs64(m.mnt_dir, &temp))
+          char buffer[32768];
+          while(getmntent_r(mtab, &m, buffer, sizeof(buffer)) != nullptr)
           {
-            // std::cout << "   " << temp.f_fsid.__val[0] << temp.f_fsid.__val[1] << " =? " << s.f_fsid.__val[0] << s.f_fsid.__val[1] << std::endl;
-            if(temp.f_type == s.f_type && (memcmp(&temp.f_fsid, &s.f_fsid, sizeof(s.f_fsid)) == 0))
+            struct statfs64 temp
+            {
+            };
+            memset(&temp, 0, sizeof(temp));
+            // std::cout << m.mnt_fsname << "," << m.mnt_dir << "," << m.mnt_type << "," << m.mnt_opts << std::endl;
+            if(0 == statfs64(m.mnt_dir, &temp))
             {
-              mountentries.emplace_back(mountentry(m.mnt_fsname, m.mnt_dir, m.mnt_type, m.mnt_opts), temp);
+              // std::cout << "   " << temp.f_fsid.__val[0] << temp.f_fsid.__val[1] << " =? " << s.f_fsid.__val[0] << s.f_fsid.__val[1] << std::endl;
+              if(temp.f_type == s.f_type && (memcmp(&temp.f_fsid, &s.f_fsid, sizeof(s.f_fsid)) == 0))
+              {
+                mountentries.emplace_back(mountentry(m.mnt_fsname, m.mnt_dir, m.mnt_type, m.mnt_opts), temp);
+              }
             }
           }
         }
-      }
 #ifndef LLFIO_COMPILING_FOR_GCOV
-      if(mountentries.empty())
-      {
-        return errc::no_such_file_or_directory;
-      }
-      /* Choose the mount entry with the most closely matching statfs. You can't choose
-      exclusively based on mount point because of bind mounts. Note that we have a
-      particular problem in Docker:
+        if(mountentries.empty())
+        {
+          return errc::no_such_file_or_directory;
+        }
+        /* Choose the mount entry with the most closely matching statfs. You can't choose
+        exclusively based on mount point because of bind mounts. Note that we have a
+        particular problem in Docker:
 
-      rootfs    /          rootfs  rw                       0 0
-      overlay   /          overlay rw,relatime,lowerdir=... 0 0
-      /dev/sda3 /etc       xfs     rw,relatime,...          0 0
-      tmpfs     /dev       tmpfs   rw,nosuid,...            0 0
-      tmpfs     /proc/acpi tmpfs   rw,nosuid,...            0 0
-      ...
+        rootfs    /          rootfs  rw                       0 0
+        overlay   /          overlay rw,relatime,lowerdir=... 0 0
+        /dev/sda3 /etc       xfs     rw,relatime,...          0 0
+        tmpfs     /dev       tmpfs   rw,nosuid,...            0 0
+        tmpfs     /proc/acpi tmpfs   rw,nosuid,...            0 0
+        ...
 
-      If f_type and f_fsid are identical for the statfs for the mount as for our file,
-      then you will get multiple mountentries, and there is no obvious way of
-      disambiguating them. What we do is match mount based on the longest match
-      of the mount point with the current path of our file.
-      */
-      if(mountentries.size() > 1)
-      {
-        OUTCOME_TRY(auto &&currentfilepath_, h.current_path());
-        string_view currentfilepath(currentfilepath_.native());
-        std::vector<std::pair<size_t, size_t>> scores(mountentries.size());
-        //std::cout << "*** For matching mount entries to file with path " << currentfilepath << ":\n";
-        for(size_t n = 0; n < mountentries.size(); n++)
+        If f_type and f_fsid are identical for the statfs for the mount as for our file,
+        then you will get multiple mountentries, and there is no obvious way of
+        disambiguating them. What we do is match mount based on the longest match
+        of the mount point with the current path of our file.
+        */
+        if(mountentries.size() > 1)
         {
-          scores[n].first =
-          (currentfilepath.substr(0, mountentries[n].first.mnt_dir.size()) == mountentries[n].first.mnt_dir) ? mountentries[n].first.mnt_dir.size() : 0;
-          //std::cout << "***    Mount entry " << mountentries[n].first.mnt_dir << " get score " << scores[n].first << std::endl;
-          scores[n].second = n;
+          OUTCOME_TRY(auto currentfilepath_, h.current_path());
+          string_view currentfilepath(currentfilepath_.native());
+          std::vector<std::pair<size_t, size_t>> scores(mountentries.size());
+          // std::cout << "*** For matching mount entries to file with path " << currentfilepath << ":\n";
+          for(size_t n = 0; n < mountentries.size(); n++)
+          {
+            scores[n].first =
+            (currentfilepath.substr(0, mountentries[n].first.mnt_dir.size()) == mountentries[n].first.mnt_dir) ? mountentries[n].first.mnt_dir.size() : 0;
+            // std::cout << "***    Mount entry " << mountentries[n].first.mnt_dir << " get score " << scores[n].first << std::endl;
+            scores[n].second = n;
+          }
+          std::sort(scores.begin(), scores.end());
+          // Choose the item whose mnt_dir most matched the current path for our file.
+          auto temp(std::move(mountentries[scores.back().second]));
+          mountentries.clear();
+          mountentries.push_back(std::move(temp));
         }
-        std::sort(scores.begin(), scores.end());
-        // Choose the item whose mnt_dir most matched the current path for our file.
-        auto temp(std::move(mountentries[scores.back().second]));
-        mountentries.clear();
-        mountentries.push_back(std::move(temp));
-      }
 #endif
-      if(!!(wanted & want::flags))
-      {
-        f_flags.rdonly = static_cast<uint32_t>((s.f_flags & MS_RDONLY) != 0);
-        f_flags.noexec = static_cast<uint32_t>((s.f_flags & MS_NOEXEC) != 0);
-        f_flags.nosuid = static_cast<uint32_t>((s.f_flags & MS_NOSUID) != 0);
-        f_flags.acls = static_cast<uint32_t>(std::string::npos != mountentries.front().first.mnt_opts.find("acl") &&
-                                             std::string::npos == mountentries.front().first.mnt_opts.find("noacl"));
-        f_flags.xattr = static_cast<uint32_t>(std::string::npos != mountentries.front().first.mnt_opts.find("xattr") &&
-                                              std::string::npos == mountentries.front().first.mnt_opts.find("nouser_xattr"));
-        //                out.f_flags.compression=0;
-        // Those filing systems supporting FALLOC_FL_PUNCH_HOLE
-        f_flags.extents = static_cast<uint32_t>(mountentries.front().first.mnt_type == "btrfs" || mountentries.front().first.mnt_type == "ext4" ||
-                                                mountentries.front().first.mnt_type == "xfs" || mountentries.front().first.mnt_type == "tmpfs");
-        ++ret;
-      }
-      if(!!(wanted & want::fstypename))
-      {
-        f_fstypename = mountentries.front().first.mnt_type;
-        ++ret;
-      }
-      if(!!(wanted & want::mntfromname))
-      {
-        f_mntfromname = mountentries.front().first.mnt_fsname;
-        ++ret;
+        if(!!(wanted & want::flags))
+        {
+          f_flags.rdonly = static_cast<uint32_t>((s.f_flags & MS_RDONLY) != 0);
+          f_flags.noexec = static_cast<uint32_t>((s.f_flags & MS_NOEXEC) != 0);
+          f_flags.nosuid = static_cast<uint32_t>((s.f_flags & MS_NOSUID) != 0);
+          f_flags.acls = static_cast<uint32_t>(std::string::npos != mountentries.front().first.mnt_opts.find("acl") &&
+                                               std::string::npos == mountentries.front().first.mnt_opts.find("noacl"));
+          f_flags.xattr = static_cast<uint32_t>(std::string::npos != mountentries.front().first.mnt_opts.find("xattr") &&
+                                                std::string::npos == mountentries.front().first.mnt_opts.find("nouser_xattr"));
+          //                out.f_flags.compression=0;
+          // Those filing systems supporting FALLOC_FL_PUNCH_HOLE
+          f_flags.extents = static_cast<uint32_t>(mountentries.front().first.mnt_type == "btrfs" || mountentries.front().first.mnt_type == "ext4" ||
+                                                  mountentries.front().first.mnt_type == "xfs" || mountentries.front().first.mnt_type == "tmpfs");
+          ++ret;
+        }
+        if(!!(wanted & want::fstypename))
+        {
+          f_fstypename = mountentries.front().first.mnt_type;
+          ++ret;
+        }
+        if(!!(wanted & want::mntfromname))
+        {
+          f_mntfromname = mountentries.front().first.mnt_fsname;
+          ++ret;
+        }
+        if(!!(wanted & want::mntonname))
+        {
+          f_mntonname = mountentries.front().first.mnt_dir;
+          ++ret;
+        }
       }
-      if(!!(wanted & want::mntonname))
+      catch(...)
       {
-        f_mntonname = mountentries.front().first.mnt_dir;
-        ++ret;
+        return error_from_exception();
       }
     }
-    catch(...)
-    {
-      return error_from_exception();
-    }
   }
 #else
   struct statfs s;
@@ -311,7 +319,176 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<size_t> statfs_t::fill(const handle &h, s
     ++ret;
   }
 #endif
+  if(!!(wanted & want::iosinprogress) || !!(wanted & want::ioswaittime))
+  {
+    OUTCOME_TRY(auto ios, _fill_ios(h, f_mntfromname));
+    if(!!(wanted & want::iosinprogress))
+    {
+      f_iosinprogress = ios.first;
+      ++ret;
+    }
+    if(!!(wanted & want::ioswaittime))
+    {
+      f_ioswaittime = ios.second;
+      ++ret;
+    }
+  }
   return ret;
 }
 
+/******************************************* statfs_t ************************************************/
+
+LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<std::pair<uint32_t, float>> statfs_t::_fill_ios(const handle &h, const std::string & /*unused*/) noexcept
+{
+  try
+  {
+#ifdef __linux__
+    struct stat s
+    {
+    };
+    memset(&s, 0, sizeof(s));
+
+    if(-1 == ::fstat(h.native_handle().fd, &s))
+    {
+      if(!h.is_symlink() || EBADF != errno)
+      {
+        return posix_error();
+      }
+      // This is a hack, but symlink_handle includes this first so there is a chicken and egg dependency problem
+      OUTCOME_TRY(detail::stat_from_symlink(s, h));
+    }
+
+    static struct last_reading_t
+    {
+      struct item
+      {
+        dev_t st_dev;
+        size_t millis{0};
+        std::chrono::steady_clock::time_point last_updated;
+
+        uint32_t f_iosinprogress{0};
+        float f_ioswaittime{0};
+      };
+      std::mutex lock;
+      std::vector<item> items;
+    } last_reading;
+    auto now = std::chrono::steady_clock::now();
+    {
+      std::lock_guard<std::mutex> g(last_reading.lock);
+      for(auto &i : last_reading.items)
+      {
+        if(i.st_dev == s.st_dev)
+        {
+          if(std::chrono::duration_cast<std::chrono::milliseconds>(now - i.last_updated) < std::chrono::milliseconds(100))
+          {
+            return {i.f_iosinprogress, i.f_ioswaittime};  // exit with old readings
+          }
+          break;
+        }
+      }
+    }
+    try
+    {
+      int fd = ::open("/proc/diskstats", O_RDONLY);
+      if(fd >= 0)
+      {
+        std::string diskstats;
+        diskstats.resize(4096);
+        for(;;)
+        {
+          auto read = ::read(fd, (char *) diskstats.data(), diskstats.size());
+          if(read < 0)
+          {
+            return posix_error();
+          }
+          if(read < (ssize_t) diskstats.size())
+          {
+            ::close(fd);
+            diskstats.resize(read);
+            break;
+          }
+          try
+          {
+            diskstats.resize(diskstats.size() << 1);
+          }
+          catch(...)
+          {
+            ::close(fd);
+            throw;
+          }
+        }
+        /* Format is (https://www.kernel.org/doc/Documentation/iostats.txt):
+        <dev id major> <dev id minor> <device name> 01 02 03 04 05 06 07 08 09 10  ...
+
+        Field 9 is i/o's currently in progress.
+        Field 10 is milliseconds spent doing i/o (cumulative).
+        */
+        auto match_line = [&](string_view sv) {
+          int major = 0, minor = 0;
+          sscanf(sv.data(), "%d %d", &major, &minor);
+          // printf("Does %d,%d match %d,%d?\n", major, minor, major(s.st_dev), minor(s.st_dev));
+          return (makedev(major, minor) == s.st_dev);
+        };
+        for(size_t is = 0, ie = diskstats.find(10); ie != diskstats.npos; is = ie + 1, ie = diskstats.find(10, is))
+        {
+          auto sv = string_view(diskstats).substr(is, ie - is);
+          if(match_line(sv))
+          {
+            int major = 0, minor = 0;
+            char devicename[64];
+            size_t fields[12];
+            sscanf(sv.data(), "%d %d %s %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu", &major, &minor, devicename, fields + 0, fields + 1, fields + 2, fields + 3,
+                   fields + 4, fields + 5, fields + 6, fields + 7, fields + 8, fields + 9);
+            std::lock_guard<std::mutex> g(last_reading.lock);
+            auto it = last_reading.items.begin();
+            for(; it != last_reading.items.end(); ++it)
+            {
+              if(it->st_dev == s.st_dev)
+              {
+                break;
+              }
+            }
+            if(it == last_reading.items.end())
+            {
+              last_reading.items.emplace_back();
+              it = --last_reading.items.end();
+              it->st_dev = s.st_dev;
+              it->millis = fields[9];
+            }
+            else
+            {
+              auto timediff = std::chrono::duration_cast<std::chrono::milliseconds>(now - it->last_updated);
+              it->f_ioswaittime = std::min((float) ((double) (fields[9] - it->millis) / timediff.count()), 1.0f);
+              it->millis = fields[9];
+            }
+            it->f_iosinprogress = (uint32_t) fields[8];
+            it->last_updated = now;
+            return {it->f_iosinprogress, it->f_ioswaittime};
+          }
+        }
+        // It's totally possible that the dev_t reported by stat()
+        // does not appear in /proc/diskstats, if this occurs then
+        // return all bits one to indicate soft failure.
+      }
+    }
+    catch(...)
+    {
+      return error_from_exception();
+    }
+#else
+    /* On FreeBSD, want::iosinprogress and want::ioswaittime could be implemented
+    using libdevstat. See https://www.freebsd.org/cgi/man.cgi?query=devstat&sektion=3.
+    Code donations welcome!
+
+    On Mac OS, getting the current i/o wait time appears to be privileged only?
+    */
+#endif
+    return {-1, _allbits1_float};
+  }
+  catch(...)
+  {
+    return error_from_exception();
+  }
+}
+
 LLFIO_V2_NAMESPACE_END
diff --git a/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp b/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp
index 60e34e5e..4da58a91 100644
--- a/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp
+++ b/include/llfio/v2.0/detail/impl/windows/directory_handle.ipp
@@ -28,6 +28,8 @@ http://www.boost.org/LICENSE_1_0.txt)
 
 #include "import.hpp"
 
+#include "../../../file_handle.hpp"
+
 LLFIO_V2_NAMESPACE_BEGIN
 
 result<directory_handle> directory_handle::directory(const path_handle &base, path_view_type path, mode _mode, creation _creation, caching _caching, flag flags) noexcept
diff --git a/include/llfio/v2.0/detail/impl/windows/file_handle.ipp b/include/llfio/v2.0/detail/impl/windows/file_handle.ipp
index 610f20b1..047a87b3 100644
--- a/include/llfio/v2.0/detail/impl/windows/file_handle.ipp
+++ b/include/llfio/v2.0/detail/impl/windows/file_handle.ipp
@@ -25,6 +25,11 @@ Distributed under the Boost Software License, Version 1.0.
 #include "../../../file_handle.hpp"
 #include "import.hpp"
 
+#include "../../../statfs.hpp"
+
+#include <mutex>
+#include <vector>
+
 LLFIO_V2_NAMESPACE_BEGIN
 
 result<file_handle> file_handle::file(const path_handle &base, file_handle::path_view_type path, file_handle::mode _mode, file_handle::creation _creation,
@@ -821,7 +826,7 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte
           }
           done = true;
         }
-        //assert(done);
+        // assert(done);
         dest_length = destoffset + extent.length;
         truncate_back_on_failure = false;
         LLFIO_DEADLINE_TO_TIMEOUT_LOOP(d);
@@ -870,4 +875,108 @@ result<file_handle::extent_type> file_handle::zero(file_handle::extent_pair exte
   return success();
 }
 
+
+/******************************************* statfs_t ************************************************/
+
+LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<std::pair<uint32_t, float>> statfs_t::_fill_ios(const handle & /*unused*/, const std::string &mntfromname) noexcept
+{
+  try
+  {
+    alignas(8) wchar_t buffer[32769];
+    // Firstly open a handle to the volume
+    OUTCOME_TRY(auto volumeh, file_handle::file({}, mntfromname, handle::mode::none, handle::creation::open_existing, handle::caching::only_metadata));
+    // Now ask the volume what physical disks it spans
+    auto *vde = reinterpret_cast<VOLUME_DISK_EXTENTS *>(buffer);
+    OVERLAPPED ol{};
+    memset(&ol, 0, sizeof(ol));
+    ol.Internal = static_cast<ULONG_PTR>(-1);
+    if(DeviceIoControl(volumeh.native_handle().h, IOCTL_VOLUME_GET_VOLUME_DISK_EXTENTS, nullptr, 0, vde, sizeof(buffer), nullptr, &ol) == 0)
+    {
+      if(ERROR_IO_PENDING == GetLastError())
+      {
+        NTSTATUS ntstat = ntwait(volumeh.native_handle().h, ol, deadline());
+        if(ntstat != 0)
+        {
+          return ntkernel_error(ntstat);
+        }
+      }
+      if(ERROR_SUCCESS != GetLastError())
+      {
+        return win32_error();
+      }
+    }
+    static struct last_reading_t
+    {
+      struct item
+      {
+        int64_t ReadTime{0}, WriteTime{0}, IdleTime{0};
+      };
+      std::mutex lock;
+      std::vector<item> items;
+    } last_reading;
+
+    uint32_t iosinprogress = 0;
+    float ioswaittime = 0;
+    DWORD disk_extents = vde->NumberOfDiskExtents;
+    for(DWORD disk_extent = 0; disk_extent < disk_extents; disk_extent++)
+    {
+      alignas(8) wchar_t physicaldrivename[32] = L"\\\\.\\PhysicalDrive", *e = physicaldrivename + 17;
+      const auto DiskNumber = vde->Extents[disk_extent].DiskNumber;
+      if(DiskNumber >= 100)
+      {
+        *e++ = '0' + ((DiskNumber / 100) % 10);
+      }
+      if(DiskNumber >= 10)
+      {
+        *e++ = '0' + ((DiskNumber / 10) % 10);
+      }
+      *e++ = '0' + (DiskNumber % 10);
+      *e = 0;
+      OUTCOME_TRY(auto diskh, file_handle::file({}, path_view(physicaldrivename, e - physicaldrivename, path_view::zero_terminated), handle::mode::none,
+                                                handle::creation::open_existing, handle::caching::only_metadata));
+      ol.Internal = static_cast<ULONG_PTR>(-1);
+      auto *dp = reinterpret_cast<DISK_PERFORMANCE *>(buffer);
+      if(DeviceIoControl(diskh.native_handle().h, IOCTL_DISK_PERFORMANCE, nullptr, 0, dp, sizeof(buffer), nullptr, &ol) == 0)
+      {
+        if(ERROR_IO_PENDING == GetLastError())
+        {
+          NTSTATUS ntstat = ntwait(diskh.native_handle().h, ol, deadline());
+          if(ntstat != 0)
+          {
+            return ntkernel_error(ntstat);
+          }
+        }
+        if(ERROR_SUCCESS != GetLastError())
+        {
+          return win32_error();
+        }
+      }
+      //printf("%llu,%llu,%llu\n", dp->ReadTime.QuadPart, dp->WriteTime.QuadPart, dp->IdleTime.QuadPart);
+      iosinprogress += dp->QueueDepth;
+      std::lock_guard<std::mutex> g(last_reading.lock);
+      if(last_reading.items.size() < DiskNumber + 1)
+      {
+        last_reading.items.resize(DiskNumber + 1);
+      }
+      else
+      {
+        uint64_t rd = (uint64_t) dp->ReadTime.QuadPart - (uint64_t) last_reading.items[DiskNumber].ReadTime;
+        uint64_t wd = (uint64_t) dp->WriteTime.QuadPart - (uint64_t) last_reading.items[DiskNumber].WriteTime;
+        uint64_t id = (uint64_t) dp->IdleTime.QuadPart - (uint64_t) last_reading.items[DiskNumber].IdleTime;
+        ioswaittime += 1 - (float) ((double) id / (rd + wd + id));
+      }
+      last_reading.items[DiskNumber].ReadTime = dp->ReadTime.QuadPart;
+      last_reading.items[DiskNumber].WriteTime = dp->WriteTime.QuadPart;
+      last_reading.items[DiskNumber].IdleTime = dp->IdleTime.QuadPart;
+    }
+    iosinprogress /= disk_extents;
+    ioswaittime /= disk_extents;
+    return {iosinprogress, std::min(ioswaittime, 1.0f)};
+  }
+  catch(...)
+  {
+    return error_from_exception();
+  }
+}
+
 LLFIO_V2_NAMESPACE_END
diff --git a/include/llfio/v2.0/detail/impl/windows/statfs.ipp b/include/llfio/v2.0/detail/impl/windows/statfs.ipp
index 8b23b4e1..82ed8ba4 100644
--- a/include/llfio/v2.0/detail/impl/windows/statfs.ipp
+++ b/include/llfio/v2.0/detail/impl/windows/statfs.ipp
@@ -141,6 +141,13 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<size_t> statfs_t::fill(const handle &h, s
     f_iosize = ffssi->PhysicalBytesPerSectorForPerformance;
     ++ret;
   }
+  if(!!(wanted & want::iosinprogress) || !!(wanted & want::ioswaittime))
+  {
+    if(f_mntfromname.empty())
+    {
+      wanted |= want::mntfromname;
+    }
+  }
   if((wanted & want::mntfromname) || (wanted & want::mntonname))
   {
     // Irrespective we need the path before figuring out the mounted device
@@ -240,6 +247,20 @@ LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<size_t> statfs_t::fill(const handle &h, s
       break;
     }
   }
+  if(!!(wanted & want::iosinprogress) || !!(wanted & want::ioswaittime))
+  {
+    OUTCOME_TRY(auto ios, _fill_ios(h, f_mntfromname));
+    if(!!(wanted & want::iosinprogress))
+    {
+      f_iosinprogress = ios.first;
+      ++ret;
+    }
+    if(!!(wanted & want::ioswaittime))
+    {
+      f_ioswaittime = ios.second;
+      ++ret;
+    }
+  }
   return ret;
 }
 
diff --git a/include/llfio/v2.0/llfio.hpp b/include/llfio/v2.0/llfio.hpp
index 88bfad2b..9cbe624d 100644
--- a/include/llfio/v2.0/llfio.hpp
+++ b/include/llfio/v2.0/llfio.hpp
@@ -34,10 +34,7 @@
 
 // If C++ Modules are on and we are not compiling the library,
 // we are either generating the interface or importing
-#if !defined(__cpp_modules) || defined(GENERATING_LLFIO_MODULE_INTERFACE) || LLFIO_DISABLE_CXX_MODULES
-// C++ Modules not on, therefore include as usual
-#define LLFIO_INCLUDE_ALL
-#else
+#if defined(__cpp_modules)
 #if defined(GENERATING_LLFIO_MODULE_INTERFACE)
 // We are generating this module's interface
 #define QUICKCPPLIB_HEADERS_ONLY 0
@@ -51,6 +48,9 @@
 import LLFIO_MODULE_NAME;
 #undef LLFIO_INCLUDE_ALL
 #endif
+#else
+// C++ Modules not on, therefore include as usual
+#define LLFIO_INCLUDE_ALL
 #endif
 
 #ifdef LLFIO_INCLUDE_ALL
@@ -63,13 +63,14 @@ import LLFIO_MODULE_NAME;
 #include "utils.hpp"
 
 #include "directory_handle.hpp"
+#include "fast_random_file_handle.hpp"
 #include "file_handle.hpp"
+//#include "io_thread_pool_group.hpp"
 #include "process_handle.hpp"
 #include "statfs.hpp"
 #ifdef LLFIO_INCLUDE_STORAGE_PROFILE
 #include "storage_profile.hpp"
 #endif
-#include "fast_random_file_handle.hpp"
 #include "symlink_handle.hpp"
 
 #include "algorithm/clone.hpp"
@@ -83,10 +84,10 @@ import LLFIO_MODULE_NAME;
 #include "algorithm/summarize.hpp"
 
 #ifndef LLFIO_EXCLUDE_MAPPED_FILE_HANDLE
+#include "mapped.hpp"
 #include "algorithm/handle_adapter/xor.hpp"
 #include "algorithm/shared_fs_mutex/memory_map.hpp"
 #include "algorithm/trivial_vector.hpp"
-#include "mapped.hpp"
 #endif
 
 #endif
diff --git a/include/llfio/v2.0/statfs.hpp b/include/llfio/v2.0/statfs.hpp
index 71732da5..06421ee4 100644
--- a/include/llfio/v2.0/statfs.hpp
+++ b/include/llfio/v2.0/statfs.hpp
@@ -1,5 +1,5 @@
 /* Information about the volume storing a file
-(C) 2015-2017 Niall Douglas <http://www.nedproductions.biz/> (8 commits)
+(C) 2015-2020 Niall Douglas <http://www.nedproductions.biz/> (8 commits)
 File Created: Dec 2015
 
 
@@ -41,13 +41,40 @@ LLFIO_V2_NAMESPACE_EXPORT_BEGIN
 
 class handle;
 
+namespace detail
+{
+  inline constexpr float constexpr_float_allbits_set_nan()
+  {
+#if defined(_MSC_VER) && !defined(__clang__)
+    // Not all bits 1, but I can't see how to do better whilst inside constexpr
+    return -NAN;
+#else
+    return -__builtin_nanf("0xffffff");  // all bits 1
+#endif
+  }
+}  // namespace detail
+
 /*! \struct statfs_t
 \brief Metadata about a filing system. Unsupported entries are all bits set.
+
+Note also that for some fields, a soft failure to read the requested value manifests
+as all bits set. For example, `f_iosinprogress` might not be computable if the
+filing system for your handle reports a `dev_t` from `fstat()` which does not
+match anything in the system's disk hardware i/o stats. As this can be completely
+benign (e.g. your handle is a socket), this is treated as a soft failure.
+
+Note for `f_iosinprogress` and `f_ioswaittime` that support is not implemented yet
+outside Microsoft Windows and Linux. Note also that for Linux, filing systems
+spanning multiple hardware devices have undefined outcomes, whereas on Windows
+you are given the average of the values for all underlying hardware devices.
+Code donations improving the support for these items on Mac OS, FreeBSD and Linux
+would be welcomed.
 */
 struct LLFIO_DECL statfs_t
 {
   static constexpr uint32_t _allbits1_32 = ~0U;
   static constexpr uint64_t _allbits1_64 = ~0ULL;
+  static constexpr float _allbits1_float = detail::constexpr_float_allbits_set_nan();
   struct f_flags_t
   {
     uint32_t rdonly : 1;             //!< Filing system is read only                                      (Windows, POSIX)
@@ -75,11 +102,31 @@ struct LLFIO_DECL statfs_t
   std::string f_mntfromname;                      /*!< mounted filesystem                 (Windows, POSIX) */
   filesystem::path f_mntonname;                   /*!< directory on which mounted         (Windows, POSIX) */
 
+  uint32_t f_iosinprogress{_allbits1_32}; /*!< i/o's currently in progress (i.e. queue depth)  (Windows, Linux) */
+  float f_ioswaittime{_allbits1_float};   /*!< percentage of time spent doing i/o (1.0 = 100%) (Windows, Linux) */
+
   //! Used to indicate what metadata should be filled in
-  QUICKCPPLIB_BITFIELD_BEGIN(want) { flags = 1 << 0, bsize = 1 << 1, iosize = 1 << 2, blocks = 1 << 3, bfree = 1 << 4, bavail = 1 << 5, files = 1 << 6, ffree = 1 << 7, namemax = 1 << 8, owner = 1 << 9, fsid = 1 << 10, fstypename = 1 << 11, mntfromname = 1 << 12, mntonname = 1 << 13, all = static_cast<unsigned>(-1) }
-  QUICKCPPLIB_BITFIELD_END(want)
+  QUICKCPPLIB_BITFIELD_BEGIN(want){flags = 1 << 0,
+                                   bsize = 1 << 1,
+                                   iosize = 1 << 2,
+                                   blocks = 1 << 3,
+                                   bfree = 1 << 4,
+                                   bavail = 1 << 5,
+                                   files = 1 << 6,
+                                   ffree = 1 << 7,
+                                   namemax = 1 << 8,
+                                   owner = 1 << 9,
+                                   fsid = 1 << 10,
+                                   fstypename = 1 << 11,
+                                   mntfromname = 1 << 12,
+                                   mntonname = 1 << 13,
+                                   iosinprogress = 1 << 14,
+                                   ioswaittime = 1 << 15,
+                                   all = static_cast<unsigned>(-1)} QUICKCPPLIB_BITFIELD_END(want)
   //! Constructs a default initialised instance (all bits set)
-  statfs_t() {}  // NOLINT  Cannot be constexpr due to lack of constexpe string default constructor :(
+  statfs_t()
+  {
+  }  // NOLINT  Cannot be constexpr due to lack of constexpe string default constructor :(
 #ifdef __cpp_exceptions
   //! Constructs a filled instance, throwing as an exception any error which might occur
   explicit statfs_t(const handle &h, want wanted = want::all)
@@ -94,6 +141,10 @@ struct LLFIO_DECL statfs_t
 #endif
   //! Fills in the structure with metadata, returning number of items filled in
   LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<size_t> fill(const handle &h, want wanted = want::all) noexcept;
+
+private:
+  // Implemented in file_handle.ipp on Windows, otherwise in statfs.ipp
+  static LLFIO_HEADERS_ONLY_MEMFUNC_SPEC result<std::pair<uint32_t, float>> _fill_ios(const handle &h, const std::string &mntfromname) noexcept;
 };
 
 LLFIO_V2_NAMESPACE_END
diff --git a/test/tests/statfs.cpp b/test/tests/statfs.cpp
new file mode 100644
index 00000000..6d4d9881
--- /dev/null
+++ b/test/tests/statfs.cpp
@@ -0,0 +1,101 @@
+/* Integration test kernel for statfs
+(C) 2020 Niall Douglas <http://www.nedproductions.biz/> (2 commits)
+File Created: Dec 2020
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License in the accompanying file
+Licence.txt or at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Distributed under the Boost Software License, Version 1.0.
+    (See accompanying file Licence.txt or copy at
+          http://www.boost.org/LICENSE_1_0.txt)
+*/
+
+#include "../test_kernel_decl.hpp"
+
+#include <future>
+#include <cmath>
+
+static inline void TestStatfsIosInProgress()
+{
+  namespace llfio = LLFIO_V2_NAMESPACE;
+  llfio::file_handle h1 = llfio::file_handle::uniquely_named_file({}, llfio::file_handle::mode::write, llfio::file_handle::caching::all,
+                                                                  llfio::file_handle::flag::unlink_on_first_close)
+                          .value();
+  llfio::file_handle h2 = llfio::file_handle::temp_file().value();
+  // h1 is within our build directory's volume, h2 is within our temp volume.
+  auto print_statfs = [](const llfio::file_handle &h, const llfio::statfs_t &statfs) {
+    std::cout << "\nFor file " << h.current_path().value() << ":";
+    std::cout << "\n fundamental filesystem block size = " << statfs.f_bsize;
+    std::cout << "\n optimal transfer block size = " << statfs.f_iosize;
+    std::cout << "\n total data blocks in filesystem = " << statfs.f_blocks;
+    std::cout << "\n free blocks in filesystem = " << statfs.f_bfree;
+    std::cout << "\n free blocks avail to non-superuser = " << statfs.f_bavail;
+    std::cout << "\n total file nodes in filesystem = " << statfs.f_files;
+    std::cout << "\n free nodes avail to non-superuser = " << statfs.f_ffree;
+    std::cout << "\n maximum filename length = " << statfs.f_namemax;
+    std::cout << "\n filesystem type name = " << statfs.f_fstypename;
+    std::cout << "\n mounted filesystem = " << statfs.f_mntfromname;
+    std::cout << "\n directory on which mounted = " << statfs.f_mntonname;
+    std::cout << "\n i/o's currently in progress (i.e. queue depth) = " << statfs.f_iosinprogress;
+    std::cout << "\n percentage of time spent doing i/o (1.0 = 100%) = " << statfs.f_ioswaittime;
+    std::cout << std::endl;
+  };
+  llfio::statfs_t s1base, s2base;
+  s1base.fill(h1).value();
+  s2base.fill(h2).value();
+  print_statfs(h1, s1base);
+  print_statfs(h2, s2base);
+  std::atomic<bool> done{false};
+  auto do_io_to = [&done](llfio::file_handle &fh) {
+    std::vector<llfio::byte> buffer;
+    buffer.resize(4096);  // 1048576
+    llfio::utils::random_fill((char *) buffer.data(), buffer.size());
+    while(!done)
+    {
+      fh.write(0, {{buffer.data(), buffer.size()}}).value();
+      fh.barrier().value();
+    }
+  };
+  {
+    auto f = std::async(std::launch::async, [&] { do_io_to(h1); });
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    llfio::statfs_t s1load, s2load;
+    s1load.fill(h1).value();
+    s2load.fill(h2).value();
+    done = true;
+    print_statfs(h1, s1load);
+    print_statfs(h2, s2load);
+    // BOOST_CHECK(s1load.f_iosinprogress > s1base.f_iosinprogress);
+    BOOST_CHECK(std::isnan(s1base.f_ioswaittime) || s1load.f_ioswaittime > s1base.f_ioswaittime);
+    f.get();
+    done = false;
+  }
+  {
+    auto f = std::async(std::launch::async, [&] { do_io_to(h2); });
+    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    llfio::statfs_t s1load, s2load;
+    s1load.fill(h1).value();
+    s2load.fill(h2).value();
+    done = true;
+    print_statfs(h1, s1load);
+    print_statfs(h2, s2load);
+    // BOOST_CHECK(s2load.f_iosinprogress > s2base.f_iosinprogress);
+    BOOST_CHECK(std::isnan(s2base.f_ioswaittime) || s2load.f_ioswaittime > s2base.f_ioswaittime);
+    f.get();
+    done = false;
+  }
+}
+
+KERNELTEST_TEST_KERNEL(integration, llfio, statfs, iosinprogress, "Tests that llfio::statfs_t::f_iosinprogress works as expected", TestStatfsIosInProgress())