Fixed quite a lot of stuff on OS X, and now the async i/o is working correctly

11 files changed, 93 insertions, 17 deletions

diff --git a/.ci.cmake b/.ci.cmake
index 28f3baee..3b134081 100644
--- a/.ci.cmake
+++ b/.ci.cmake
@@ -68,9 +68,9 @@ else()
       COMMAND cp -a release_notes.md afio/
       COMMAND cp -a --parents prebuilt/lib/libafio_sl-2.0-Darwin-x86_64-Release.a afio/
       COMMAND cp -a --parents prebuilt/lib/libafio_dl-2.0-Darwin-x86_64-Release.so afio/
-      COMMAND "${CMAKE_COMMAND}" -E tar cfz afio-v2.0-binaries-Darwin.tgz afio
+      COMMAND "${CMAKE_COMMAND}" -E tar cfz afio-v2.0-binaries-darwin64.tgz afio
     )
-    get_filename_component(toupload afio-v2.0-binaries-Darwin.tgz ABSOLUTE)
+    get_filename_component(toupload afio-v2.0-binaries-darwin64.tgz ABSOLUTE)
   endif()
 endif()
 set(retval2 0)
diff --git a/.travis.yml b/.travis.yml
index 91acc839..a7e6cb28 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -121,8 +121,8 @@ after_success:
       curl -T $NEWNAME -u jenkins-nedprod:$JENKINS_NEDPROD_PASSWORD https://dedi4.nedprod.com/static/files/upload/; fi;
     fi
   - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then   
-      NEWNAME=afio-v2.0-binaries-Darwin-$TRAVIS_COMMIT.tgz;
-      mv afio-v2.0-binaries-Darwin.tgz $NEWNAME;
+      NEWNAME=afio-v2.0-binaries-darwin64-$TRAVIS_COMMIT.tgz;
+      mv afio-v2.0-binaries-darwin64.tgz $NEWNAME;
       curl -T $NEWNAME -u jenkins-nedprod:$JENKINS_NEDPROD_PASSWORD https://dedi4.nedprod.com/static/files/upload/; fi;
     fi
 
diff --git a/Readme.md b/Readme.md
index 3d6bcf7b..21b90a46 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,10 +1,11 @@
 This is the post-peer-review AFIO v2 rewrite. You can view its documentation at https://ned14.github.io/afio/
 
-<b>master branch test status</b> Linux: [![Build Status](https://travis-ci.org/ned14/afio.svg?branch=master)](https://travis-ci.org/ned14/afio) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/ox59o2r276xbmef7/branch/master?svg=true)](https://ci.appveyor.com/project/ned14/afio/branch/master) <b>CMake dashboard</b>: http://my.cdash.org/index.php?project=Boost.AFIO
+<b>master branch test status</b> Linux & OS X: [![Build Status](https://travis-ci.org/ned14/afio.svg?branch=master)](https://travis-ci.org/ned14/afio) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/ox59o2r276xbmef7/branch/master?svg=true)](https://ci.appveyor.com/project/ned14/afio/branch/master) <b>CMake dashboard</b>: http://my.cdash.org/index.php?project=Boost.AFIO
 
-Tarballs of source and prebuilt binaries for Linux x64 and Windows x64:
+Tarballs of source and prebuilt binaries for Linux x64, MacOS x64 and Windows x64:
 - https://dedi4.nedprod.com/static/files/afio-v2.0-source-latest.tar.xz
 - https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-linux64-latest.tgz
+- https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-darwin64-latest.tgz
 - https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-win64-latest.zip
 
 
diff --git a/include/afio/revision.hpp b/include/afio/revision.hpp
index e4be7b9d..50c08022 100644
--- a/include/afio/revision.hpp
+++ b/include/afio/revision.hpp
@@ -1,4 +1,4 @@
 // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
-#define AFIO_PREVIOUS_COMMIT_REF    9029010705b5ac83ad73ce81e8a948de6fcd10ab
-#define AFIO_PREVIOUS_COMMIT_DATE   "2018-02-01 09:21:12 +00:00"
-#define AFIO_PREVIOUS_COMMIT_UNIQUE 90290107
+#define AFIO_PREVIOUS_COMMIT_REF    bb31a246e0ee748d2379be0159d657c5f7792cbb
+#define AFIO_PREVIOUS_COMMIT_DATE   "2018-02-01 09:51:25 +00:00"
+#define AFIO_PREVIOUS_COMMIT_UNIQUE bb31a246
diff --git a/include/afio/v2.0/async_file_handle.hpp b/include/afio/v2.0/async_file_handle.hpp
index 2adedc2d..f10b012d 100644
--- a/include/afio/v2.0/async_file_handle.hpp
+++ b/include/afio/v2.0/async_file_handle.hpp
@@ -414,6 +414,12 @@ public:
   }
 
   /*! \brief Schedule a read to occur asynchronously.
+   
+  Note that some OS kernels can only process a limited number async i/o
+  operations at a time. You should therefore check for the error `std::errc::resource_unavailable_try_again`
+  and gracefully reschedule the i/o for a later time. This temporary
+  failure may be returned immediately, or to the completion handler
+  and hence you ought to handle both situations.
 
   \return Either an io_state_ptr to the i/o in progress, or an error code.
   \param reqs A scatter-gather and offset request.
@@ -451,6 +457,13 @@ public:
 
   /*! \brief Schedule a write to occur asynchronously.
 
+  Note that some OS kernels can only process a limited number async i/o
+  operations at a time. You should therefore check for the error `std::errc::resource_unavailable_try_again`
+  and gracefully reschedule the i/o for a later time. This temporary
+  failure may be returned immediately, or to the completion handler
+  and hence you ought to handle both situations.
+
+   
   \return Either an io_state_ptr to the i/o in progress, or an error code.
   \param reqs A scatter-gather and offset request.
   \param completion A callable to call upon i/o completion. Spec is `void(async_file_handle *, io_result<const_buffers_type> &)`.
diff --git a/include/afio/v2.0/config.hpp b/include/afio/v2.0/config.hpp
index dd19fcf0..a79ccfd8 100644
--- a/include/afio/v2.0/config.hpp
+++ b/include/afio/v2.0/config.hpp
@@ -363,7 +363,7 @@ public:
   /* NOTE TO SELF: The error_info constructor implementation is in handle.hpp as we need that
   defined before we can do useful logging.
   */
-  //! Construct from an error condition enum
+  //! Implicit construct from an error condition enum
   OUTCOME_TEMPLATE(class ErrorCondEnum)
   OUTCOME_TREQUIRES(OUTCOME_TPRED(std::is_error_condition_enum<ErrorCondEnum>::value))
   error_info(ErrorCondEnum &&v)  // NOLINT
@@ -391,6 +391,30 @@ inline bool operator!=(const error_info &a, const error_info &b)
 {
   return a.ec != b.ec;
 }
+OUTCOME_TEMPLATE(class ErrorCondEnum)
+OUTCOME_TREQUIRES(OUTCOME_TPRED(std::is_error_condition_enum<ErrorCondEnum>::value))
+inline bool operator==(const error_info &a, const ErrorCondEnum &b)
+{
+  return a.ec == std::error_condition(b);
+}
+OUTCOME_TEMPLATE(class ErrorCondEnum)
+OUTCOME_TREQUIRES(OUTCOME_TPRED(std::is_error_condition_enum<ErrorCondEnum>::value))
+inline bool operator==(const ErrorCondEnum &a, const error_info &b)
+{
+  return std::error_condition(a) == b.ec;
+}
+OUTCOME_TEMPLATE(class ErrorCondEnum)
+OUTCOME_TREQUIRES(OUTCOME_TPRED(std::is_error_condition_enum<ErrorCondEnum>::value))
+inline bool operator!=(const error_info &a, const ErrorCondEnum &b)
+{
+  return a.ec != std::error_condition(b);
+}
+OUTCOME_TEMPLATE(class ErrorCondEnum)
+OUTCOME_TREQUIRES(OUTCOME_TPRED(std::is_error_condition_enum<ErrorCondEnum>::value))
+inline bool operator!=(const ErrorCondEnum &a, const error_info &b)
+{
+  return std::error_condition(a) != b.ec;
+}
 #ifndef NDEBUG
 // Is trivial in all ways, except default constructibility
 static_assert(std::is_trivially_copyable<error_info>::value, "error_info is not a trivially copyable!");
diff --git a/include/afio/v2.0/detail/impl/posix/async_file_handle.ipp b/include/afio/v2.0/detail/impl/posix/async_file_handle.ipp
index c1ea2faf..7f259a3d 100644
--- a/include/afio/v2.0/detail/impl/posix/async_file_handle.ipp
+++ b/include/afio/v2.0/detail/impl/posix/async_file_handle.ipp
@@ -179,8 +179,28 @@ template <class BuffersType, class IORoutine> result<async_file_handle::io_state
   }
   size_t items(reqs.buffers.size());
 #if AFIO_USE_POSIX_AIO && defined(AIO_LISTIO_MAX)
+  // If this i/o could never be done atomically, reject
   if(items > AIO_LISTIO_MAX)
     return std::errc::invalid_argument;
+#if defined(__FreeBSD__) || defined(__APPLE__)
+  if(!service()->using_kqueues())
+  {
+    // BSD and Apple put a tight limit on how many entries
+    // aio_suspend() will take in order to have reasonable
+    // performance. But their documentation lies, if you
+    // feed more than AIO_LISTIO_MAX items to aio_suspend
+    // it does NOT return EINVAL as specified, but rather
+    // simply marks all items past AIO_LISTIO_MAX as failed
+    // with EAGAIN. That punishes performance for AFIO
+    // because we loop setting up and tearing down
+    // the handlers, so if we would overload afio_suspend,
+    // better to error out now rather that later in io_service.
+    if(service()->_aiocbsv.size() + items > AIO_LISTIO_MAX)
+    {
+      return std::errc::resource_unavailable_try_again;
+    }
+  }
+#endif
 #endif
   bool must_deallocate_self = false;
   if(mem.empty())
diff --git a/include/afio/v2.0/detail/impl/posix/io_handle.ipp b/include/afio/v2.0/detail/impl/posix/io_handle.ipp
index 68b67ee5..6f88614e 100644
--- a/include/afio/v2.0/detail/impl/posix/io_handle.ipp
+++ b/include/afio/v2.0/detail/impl/posix/io_handle.ipp
@@ -39,6 +39,9 @@ size_t io_handle::max_buffers() const noexcept
   static size_t v;
   if(v == 0u)
   {
+#ifdef __APPLE__
+    v = 1;
+#else
     long r = sysconf(_SC_IOV_MAX);
     if(r == -1)
     {
@@ -49,6 +52,7 @@ size_t io_handle::max_buffers() const noexcept
 #endif
     }
     v = r;
+#endif
   }
   return v;
 }
diff --git a/include/afio/v2.0/io_handle.hpp b/include/afio/v2.0/io_handle.hpp
index 2f2b87aa..a9732fac 100644
--- a/include/afio/v2.0/io_handle.hpp
+++ b/include/afio/v2.0/io_handle.hpp
@@ -194,7 +194,7 @@ public:
   but other OSs do not. Some OSs guarantee that each i/o syscall has effects atomically visible or not
   to other i/o, other OSs do not.
 
-  Microsoft Windows does not implement scatter-gather file i/o syscalls except for unbuffered i/o.
+  Microsoft Windows and OS X does not implement scatter-gather file i/o syscalls.
   Thus this function will always return `1` in that situation.
   */
   AFIO_HEADERS_ONLY_VIRTUAL_SPEC size_t max_buffers() const noexcept;
diff --git a/release_notes.md b/release_notes.md
index 92c081f5..714996a9 100644
--- a/release_notes.md
+++ b/release_notes.md
@@ -1,4 +1,4 @@
-<center><table border="0" cellpadding="4">
+<center><table border="0" cellpadding="4">
 <tr>
 <td align="center"> <a href="https://github.com/ned14/afio">AFIO</a><br><a href="https://github.com/ned14/afio">on GitHub</a> </td>
 <td align="center"> <a href="http://my.cdash.org/index.php?project=Boost.AFIO">CTest summary</a><br><a href="http://my.cdash.org/index.php?project=Boost.AFIO">dashboard</a> </td>
@@ -6,6 +6,7 @@
 <td align="center"> <a href="https://ci.appveyor.com/project/ned14/afio/branch/master">Windows CI:</a><img src="https://ci.appveyor.com/api/projects/status/680b1pt9srnoprs3/branch/master?svg=true"/> </td>
 <td align="center"> <a href="https://dedi4.nedprod.com/static/files/afio-v2.0-source-latest.tar.xz">Latest stable</a><br><a href="https://dedi4.nedprod.com/static/files/afio-v2.0-source-latest.tar.xz">sources</a> </td>
 <td align="center"> <a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-linux64-latest.tgz">Latest stable</a><br><a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-linux64-latest.tgz">Linux x64 prebuilt</a> </td>
+<td align="center"> <a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-darwin-latest.tgz">Latest stable</a><br><a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-darwin64-latest.tgz">OS X x64 prebuilt</a> </td>
 <td align="center"> <a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-win64-latest.zip">Latest stable</a><br/><a href="https://dedi4.nedprod.com/static/files/afio-v2.0-binaries-win64-latest.zip">VS2017 x64 prebuilt</a> </td>
 </tr>
 </table></center>
@@ -82,6 +83,7 @@ co_await co_write(fh, {{{buffer, sizeof(buffer)}}, 0}).value();
 These compilers and OS are regularly tested:
 - GCC 7.0 (Linux 4,x x64)
 - clang 4.0 (Linux 4.x x64)
+- clang 5.0 (OS X 10.12 x64)
 - Visual Studio 2017 (Windows 10 x64)
 
 Other compilers, architectures and OSs may work, but are not tested regularly. You will need a Filesystem TS
@@ -188,10 +190,10 @@ Todo thereafter in order of priority:
 
 | NEW in v2 | Windows | POSIX |     |
 | --------- | --------| ----- | --- |
-| ✔ |   |   | Linux KAIO support for native non-blocking `O_DIRECT` i/o
 | ✔ |   |   | Reliable directory hierarchy deletion algorithm.
 | ✔ |   |   | Reliable directory hierarchy copy algorithm.
 | ✔ |   |   | Reliable directory hierarchy update (two and three way) algorithm.
+| ✔ |   |   | Linux KAIO support for native non-blocking `O_DIRECT` i/o
 | ✔ |   |   | `std::pmr::memory_resource` adapting a file backing if on C++ 17.
 | ✔ |   |   | Extended attributes support.
 | ✔ |   |   | Algorithm to replace all duplicate content with hard links.
diff --git a/test/tests/async_io.cpp b/test/tests/async_io.cpp
index 0d9ffaec..ed202d11 100644
--- a/test/tests/async_io.cpp
+++ b/test/tests/async_io.cpp
@@ -39,11 +39,16 @@ static inline void TestAsyncFileHandle()
   afio::async_file_handle::const_buffer_type bt{buffer, sizeof(buffer)};  // NOLINT
   for(size_t n = 0; n < 1024; n++)
   {
+  retry:
     std::promise<afio::async_file_handle::const_buffers_type> p;
     auto f(p.get_future());
-    auto g(h
+    auto schedule_io = [&]{ return h
            .async_write({bt, n * 4096}, [ p = std::move(p), n ](afio::async_file_handle *, afio::async_file_handle::io_result<afio::async_file_handle::const_buffers_type> & result) mutable {
              (void) n;
+             if(!result && result.error() == std::errc::resource_unavailable_try_again)
+             {
+               std::cout << "*** Completion handler saw error " << result.error() << std::endl;
+             }
              try
              {
                p.set_value(result.value());
@@ -54,9 +59,16 @@ static inline void TestAsyncFileHandle()
                p.set_exception(std::current_exception());
                // std::cout << "Written block " << n << " unsuccessfully" << std::endl;
              }
-           })
-           .value());
-    futures.emplace_back(std::move(f), std::move(g));
+           }); };
+    auto g(schedule_io());
+    if(!g && g.error() == std::errc::resource_unavailable_try_again)
+    {
+      // Sleep until at least i/o is processed
+      service.run().value();
+      goto retry;
+    }
+    auto i(std::move(g).value());
+    futures.emplace_back(std::move(f), std::move(i));
   }
   // Pump the i/o until no more work remains.
   while(service.run().value())