diff options
| -rw-r--r-- | CMakeLists.txt | 8 | ||||
| -rw-r--r-- | cmake/headers.cmake | 1 | ||||
| -rw-r--r-- | include/llfio/revision.hpp | 6 | ||||
| -rw-r--r-- | include/llfio/v2.0/detail/impl/io_multiplexer.ipp | 6 | ||||
| -rw-r--r-- | include/llfio/v2.0/detail/impl/windows/io_multiplexer.ipp | 111 | ||||
| -rw-r--r-- | include/llfio/v2.0/detail/impl/windows/test/iocp_multiplexer.ipp | 115 | ||||
| -rw-r--r-- | include/llfio/v2.0/io_handle.hpp | 98 | ||||
| -rw-r--r-- | include/llfio/v2.0/io_multiplexer.hpp | 238 |
8 files changed, 371 insertions, 212 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c3010e7a..3866d3a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -194,6 +194,14 @@ int main() { } " CXX_HAS_CXX17_FILESYSTEM) if(NOT CXX_HAS_CXX17_FILESYSTEM) + check_cxx_source_linkage(" +#include <experimental/filesystem> +int main() { + try { return std::experimental::filesystem::path(\"hi\").empty(); } catch(std::experimental::filesystem::filesystem_error) { return 1; } +} +" CXX_HAS_CXX_EXPERIMENTAL_FILESYSTEM) +endif() +if(NOT CXX_HAS_CXX17_FILESYSTEM AND NOT CXX_HAS_CXX_EXPERIMENTAL_FILESYSTEM) indented_message(STATUS "NOTE: Standard <filesystem> not found in the current compiler configuration (try forcing C++ 17 or later?), attempting to figure out what <experimental/filesystem> linker flags to use for this STL ...") # Are we on libstdc++ or libc++? check_cxx_source_compiles(" diff --git a/cmake/headers.cmake b/cmake/headers.cmake index 116cc3a9..759bae0d 100644 --- a/cmake/headers.cmake +++ b/cmake/headers.cmake @@ -63,6 +63,7 @@ set(llfio_HEADERS "include/llfio/v2.0/detail/impl/windows/statfs.ipp" "include/llfio/v2.0/detail/impl/windows/storage_profile.ipp" "include/llfio/v2.0/detail/impl/windows/symlink_handle.ipp" + "include/llfio/v2.0/detail/impl/windows/test/iocp_multiplexer.ipp" "include/llfio/v2.0/detail/impl/windows/utils.ipp" "include/llfio/v2.0/directory_handle.hpp" "include/llfio/v2.0/fast_random_file_handle.hpp" diff --git a/include/llfio/revision.hpp b/include/llfio/revision.hpp index 1f9017b8..086ec045 100644 --- a/include/llfio/revision.hpp +++ b/include/llfio/revision.hpp @@ -1,4 +1,4 @@ // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time -#define LLFIO_PREVIOUS_COMMIT_REF 48d74e82e05e1756467441de4e916996c1a9b4be -#define LLFIO_PREVIOUS_COMMIT_DATE "2020-04-10 09:49:07 +00:00" -#define LLFIO_PREVIOUS_COMMIT_UNIQUE 48d74e82 +#define LLFIO_PREVIOUS_COMMIT_REF 7e8bbc8b8d4b2822d9ac36b546d68586dc8e7d82 +#define LLFIO_PREVIOUS_COMMIT_DATE "2020-04-15 08:59:12 +00:00" +#define LLFIO_PREVIOUS_COMMIT_UNIQUE 7e8bbc8b diff --git a/include/llfio/v2.0/detail/impl/io_multiplexer.ipp b/include/llfio/v2.0/detail/impl/io_multiplexer.ipp index 04f1e888..a6afd0f0 100644 --- a/include/llfio/v2.0/detail/impl/io_multiplexer.ipp +++ b/include/llfio/v2.0/detail/impl/io_multiplexer.ipp @@ -54,8 +54,10 @@ template <> struct io_multiplexer_impl<true> : io_multiplexer LLFIO_V2_NAMESPACE_END +#if LLFIO_ENABLE_TEST_IO_MULTIPLEXERS #if defined(_WIN32) -#include "windows/io_multiplexer.ipp" +#include "windows/test/iocp_multiplexer.ipp" #else -//#include "posix/io_multiplexer.ipp" +//#include "posix/test/io_uring_multiplexer.ipp" +#endif #endif diff --git a/include/llfio/v2.0/detail/impl/windows/io_multiplexer.ipp b/include/llfio/v2.0/detail/impl/windows/io_multiplexer.ipp deleted file mode 100644 index b4b971f2..00000000 --- a/include/llfio/v2.0/detail/impl/windows/io_multiplexer.ipp +++ /dev/null @@ -1,111 +0,0 @@ -/* Multiplex file i/o -(C) 2019 Niall Douglas <http://www.nedproductions.biz/> (9 commits) -File Created: Nov 2019 - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License in the accompanying file -Licence.txt or at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - -Distributed under the Boost Software License, Version 1.0. - (See accompanying file Licence.txt or copy at - http://www.boost.org/LICENSE_1_0.txt) -*/ - -#include "../../../io_multiplexer.hpp" - -#ifndef _WIN32 -#error This implementation file is for Microsoft Windows only -#endif - -#include "import.hpp" - -LLFIO_V2_NAMESPACE_BEGIN - -template <bool is_threadsafe> struct win_iocp_multiplexer final : io_multiplexer_impl<is_threadsafe> -{ - using _base = io_multiplexer_impl<is_threadsafe>; - using _lock_guard = typename _base::_lock_guard; - - explicit win_iocp_multiplexer(size_t threads) { (void) threads; } - virtual ~win_iocp_multiplexer() - { - if(_v) - { - (void) win_iocp_multiplexer::close(); - } - } - // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<path_type> current_path() const noexcept override; - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> close() noexcept override { return _base::close(); } - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC native_handle_type release() noexcept override { return _base::release(); } - - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<uint8_t> do_io_handle_register(io_handle *h) noexcept override { - windows_nt_kernel::init(); - using namespace windows_nt_kernel; - LLFIO_LOG_FUNCTION_CALL(this); - IO_STATUS_BLOCK isb = make_iostatus(); - FILE_COMPLETION_INFORMATION fci{}; - memset(&fci, 0, sizeof(fci)); - fci.Port = this->_v.h; - fci.Key = nullptr; - NTSTATUS ntstat = NtSetInformationFile(h->native_handle().h, &isb, &fci, sizeof(fci), FileCompletionInformation); - if(STATUS_PENDING == ntstat) - { - ntstat = ntwait(h->native_handle().h, isb, deadline()); - } - if(ntstat < 0) - { - return ntkernel_error(ntstat); - } - // If this works, we can avoid IOCP entirely for immediately completing i/o - return (uint8_t) !SetFileCompletionNotificationModes(h->native_handle().h, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS | FILE_SKIP_SET_EVENT_ON_HANDLE); - } - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> do_io_handle_deregister(io_handle *h) noexcept override { - windows_nt_kernel::init(); - using namespace windows_nt_kernel; - LLFIO_LOG_FUNCTION_CALL(this); - IO_STATUS_BLOCK isb = make_iostatus(); - FILE_COMPLETION_INFORMATION fci{}; - memset(&fci, 0, sizeof(fci)); - fci.Port = nullptr; - fci.Key = nullptr; - NTSTATUS ntstat = NtSetInformationFile(h->native_handle().h, &isb, &fci, sizeof(fci), FileReplaceCompletionInformation); - if(STATUS_PENDING == ntstat) - { - ntstat = ntwait(h->native_handle().h, isb, deadline()); - } - if(ntstat < 0) - { - return ntkernel_error(ntstat); - } - return success(); - } - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC size_t do_io_handle_max_buffers(const io_handle *h) const noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<registered_buffer_type> do_io_handle_allocate_registered_buffer(io_handle *h, size_t &bytes) noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<buffers_type> do_io_handle_read(io_handle *h, io_request<buffers_type> reqs, deadline d) noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<buffers_type> do_io_handle_read(io_handle *h, registered_buffer_type base, io_request<buffers_type> reqs, deadline d) noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> do_io_handle_write(io_handle *h, io_request<const_buffers_type> reqs, deadline d) noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> do_io_handle_write(io_handle *h, registered_buffer_type base, io_request<const_buffers_type> reqs, deadline d) noexcept override {} - //LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> do_io_handle_barrier(io_handle *h, io_request<const_buffers_type> reqs, barrier_kind kind, deadline d) noexcept override {} -}; - -LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_win_iocp(size_t threads) noexcept -{ - if(1 == threads) - { - return std::make_unique<win_iocp_multiplexer<false>>(1); - } - return std::make_unique<win_iocp_multiplexer<true>>(threads); -} - -LLFIO_V2_NAMESPACE_END diff --git a/include/llfio/v2.0/detail/impl/windows/test/iocp_multiplexer.ipp b/include/llfio/v2.0/detail/impl/windows/test/iocp_multiplexer.ipp new file mode 100644 index 00000000..cb974f2f --- /dev/null +++ b/include/llfio/v2.0/detail/impl/windows/test/iocp_multiplexer.ipp @@ -0,0 +1,115 @@ +/* Multiplex file i/o +(C) 2019 Niall Douglas <http://www.nedproductions.biz/> (9 commits) +File Created: Nov 2019 + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License in the accompanying file +Licence.txt or at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Distributed under the Boost Software License, Version 1.0. + (See accompanying file Licence.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +*/ + +#include "../../../../io_multiplexer.hpp" + +#ifndef _WIN32 +#error This implementation file is for Microsoft Windows only +#endif + +#include "../import.hpp" + +LLFIO_V2_NAMESPACE_BEGIN + +namespace test +{ + template <bool is_threadsafe> struct win_iocp_multiplexer final : io_multiplexer_impl<is_threadsafe> + { + using _base = io_multiplexer_impl<is_threadsafe>; + using _lock_guard = typename _base::_lock_guard; + + explicit win_iocp_multiplexer(size_t threads) { (void) threads; } + virtual ~win_iocp_multiplexer() + { + if(_v) + { + (void) win_iocp_multiplexer::close(); + } + } + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<path_type> current_path() const noexcept override; + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> close() noexcept override { return _base::close(); } + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC native_handle_type release() noexcept override { return _base::release(); } + + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<uint8_t> do_io_handle_register(io_handle *h) noexcept override + { + windows_nt_kernel::init(); + using namespace windows_nt_kernel; + LLFIO_LOG_FUNCTION_CALL(this); + IO_STATUS_BLOCK isb = make_iostatus(); + FILE_COMPLETION_INFORMATION fci{}; + memset(&fci, 0, sizeof(fci)); + fci.Port = this->_v.h; + fci.Key = nullptr; + NTSTATUS ntstat = NtSetInformationFile(h->native_handle().h, &isb, &fci, sizeof(fci), FileCompletionInformation); + if(STATUS_PENDING == ntstat) + { + ntstat = ntwait(h->native_handle().h, isb, deadline()); + } + if(ntstat < 0) + { + return ntkernel_error(ntstat); + } + // If this works, we can avoid IOCP entirely for immediately completing i/o + return (uint8_t) !SetFileCompletionNotificationModes(h->native_handle().h, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS | FILE_SKIP_SET_EVENT_ON_HANDLE); + } + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> do_io_handle_deregister(io_handle *h) noexcept override + { + windows_nt_kernel::init(); + using namespace windows_nt_kernel; + LLFIO_LOG_FUNCTION_CALL(this); + IO_STATUS_BLOCK isb = make_iostatus(); + FILE_COMPLETION_INFORMATION fci{}; + memset(&fci, 0, sizeof(fci)); + fci.Port = nullptr; + fci.Key = nullptr; + NTSTATUS ntstat = NtSetInformationFile(h->native_handle().h, &isb, &fci, sizeof(fci), FileReplaceCompletionInformation); + if(STATUS_PENDING == ntstat) + { + ntstat = ntwait(h->native_handle().h, isb, deadline()); + } + if(ntstat < 0) + { + return ntkernel_error(ntstat); + } + return success(); + } + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC size_t do_io_handle_max_buffers(const io_handle *h) const noexcept override {} + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<registered_buffer_type> do_io_handle_allocate_registered_buffer(io_handle *h, size_t &bytes) noexcept override {} + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC void begin_io_operation(io_operation_state *op) noexcept; // default implementation is in io_handle.hpp + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC bool check_io_operation(io_operation_state *op) noexcept { return false; } + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<wait_for_completed_io_statistics> check_for_any_completed_io(deadline d = std::chrono::seconds(0), size_t max_completions = (size_t) -1) noexcept { return success(); } + // LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> wake_check_for_any_completed_io() noexcept { return success(); } + }; + + LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_win_iocp(size_t threads) noexcept + { + if(1 == threads) + { + return std::make_unique<win_iocp_multiplexer<false>>(1); + } + return std::make_unique<win_iocp_multiplexer<true>>(threads); + } +} // namespace test + +LLFIO_V2_NAMESPACE_END diff --git a/include/llfio/v2.0/io_handle.hpp b/include/llfio/v2.0/io_handle.hpp index f692e6c2..593476da 100644 --- a/include/llfio/v2.0/io_handle.hpp +++ b/include/llfio/v2.0/io_handle.hpp @@ -165,6 +165,37 @@ protected: //! The virtualised implementation of `barrier()` used if no multiplexer has been set. LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> _do_barrier(io_request<const_buffers_type> reqs, barrier_kind kind, deadline d) noexcept; + io_result<buffers_type> _do_multiplexer_read(registered_buffer_type &&base, io_request<buffers_type> reqs, deadline d) noexcept + { + io_multiplexer::io_operation_state state(this, std::move(base), d, std::move(reqs)); + _ctx->begin_io_operation(&state); + while(!is_finished(_ctx->check_io_operation(&state))) + { + OUTCOME_TRY(_ctx->check_for_any_completed_io({})); + } + return std::move(state.payload.completed_read); + } + io_result<const_buffers_type> _do_multiplexer_write(registered_buffer_type &&base, io_request<const_buffers_type> reqs, deadline d) noexcept + { + io_multiplexer::io_operation_state state(this, std::move(base), d, std::move(reqs)); + _ctx->begin_io_operation(&state); + while(!is_finished(_ctx->check_io_operation(&state))) + { + OUTCOME_TRY(_ctx->check_for_any_completed_io({})); + } + return std::move(state.payload.completed_write_or_barrier); + } + io_result<const_buffers_type> _do_multiplexer_barrier(registered_buffer_type &&base, io_request<const_buffers_type> reqs, barrier_kind kind, deadline d) noexcept + { + io_multiplexer::io_operation_state state(this, std::move(base), d, std::move(reqs), kind); + _ctx->begin_io_operation(&state); + while(!is_finished(_ctx->check_io_operation(&state))) + { + OUTCOME_TRY(_ctx->check_for_any_completed_io({})); + } + return std::move(state.payload.completed_write_or_barrier); + } + public: /*! \brief The *maximum* number of buffers which a single read or write syscall can (atomically) process at a time for this specific open handle. On POSIX, this is known as `IOV_MAX`. @@ -246,28 +277,10 @@ public: The asynchronous implementation in async_file_handle performs one calloc and one free. */ LLFIO_MAKE_FREE_FUNCTION - io_result<buffers_type> read(io_request<buffers_type> reqs, deadline d = deadline()) noexcept - { - if(_ctx == nullptr) - { - return _do_read(reqs, d); - } - io_multiplexer::io_operation_state state(this, {}, d, std::move(reqs)); - _ctx->do_io_operation(&state); - return std::move(state.payload.completed_read); - } + io_result<buffers_type> read(io_request<buffers_type> reqs, deadline d = deadline()) noexcept { return (_ctx == nullptr) ? _do_read(reqs, d) : _do_multiplexer_read({}, reqs, d); } //! \overload Registered buffer overload, scatter list **must** be wholly within the registered buffer LLFIO_MAKE_FREE_FUNCTION - io_result<buffers_type> read(registered_buffer_type base, io_request<buffers_type> reqs, deadline d = deadline()) noexcept - { - if(_ctx == nullptr) - { - return _do_read(std::move(base), reqs, d); - } - io_multiplexer::io_operation_state state(this, std::move(base), d, std::move(reqs)); - _ctx->do_io_operation(&state); - return std::move(state.payload.completed_read); - } + io_result<buffers_type> read(registered_buffer_type base, io_request<buffers_type> reqs, deadline d = deadline()) noexcept { return (_ctx == nullptr) ? _do_read(std::move(base), reqs, d) : _do_multiplexer_read(std::move(base), reqs, d); } //! \overload Convenience initialiser list based overload for `read()` LLFIO_MAKE_FREE_FUNCTION io_result<size_type> read(extent_type offset, std::initializer_list<buffer_type> lst, deadline d = deadline()) noexcept @@ -307,28 +320,10 @@ public: The asynchronous implementation in async_file_handle performs one calloc and one free. */ LLFIO_MAKE_FREE_FUNCTION - io_result<const_buffers_type> write(io_request<const_buffers_type> reqs, deadline d = deadline()) noexcept - { - if(_ctx == nullptr) - { - return _do_write(reqs, d); - } - io_multiplexer::io_operation_state state(this, {}, d, std::move(reqs)); - _ctx->do_io_operation(&state); - return std::move(state.payload.completed_write_or_barrier); - } + io_result<const_buffers_type> write(io_request<const_buffers_type> reqs, deadline d = deadline()) noexcept { return (_ctx == nullptr) ? _do_write(reqs, d) : _do_multiplexer_write({}, std::move(reqs), d); } //! \overload Registered buffer overload, gather list **must** be wholly within the registered buffer LLFIO_MAKE_FREE_FUNCTION - io_result<const_buffers_type> write(registered_buffer_type base, io_request<const_buffers_type> reqs, deadline d = deadline()) noexcept - { - if(_ctx == nullptr) - { - return _do_write(std::move(base), reqs, d); - } - io_multiplexer::io_operation_state state(this, std::move(base), d, std::move(reqs)); - _ctx->do_io_operation(&state); - return std::move(state.payload.completed_write_or_barrier); - } + io_result<const_buffers_type> write(registered_buffer_type base, io_request<const_buffers_type> reqs, deadline d = deadline()) noexcept { return (_ctx == nullptr) ? _do_write(std::move(base), reqs, d) : _do_multiplexer_write(std::move(base), std::move(reqs), d); } //! \overload Convenience initialiser list based overload for `write()` LLFIO_MAKE_FREE_FUNCTION io_result<size_type> write(extent_type offset, std::initializer_list<const_buffer_type> lst, deadline d = deadline()) noexcept @@ -378,13 +373,7 @@ public: LLFIO_MAKE_FREE_FUNCTION LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_result<const_buffers_type> barrier(io_request<const_buffers_type> reqs = io_request<const_buffers_type>(), barrier_kind kind = barrier_kind::nowait_data_only, deadline d = deadline()) noexcept { - if(_ctx == nullptr) - { - return _do_barrier(reqs, kind, d); - } - io_multiplexer::io_operation_state state(this, {}, d, std::move(reqs), kind); - _ctx->do_io_operation(&state); - return std::move(state.payload.completed_write_or_barrier); + return (_ctx == nullptr) ? _do_barrier(reqs, kind, d) : _do_multiplexer_barrier({}, std::move(reqs), kind, d); } //! \overload Convenience overload LLFIO_MAKE_FREE_FUNCTION @@ -402,23 +391,24 @@ inline result<io_multiplexer::registered_buffer_type> io_multiplexer::do_io_hand { return h->_do_allocate_registered_buffer(bytes); } -inline void io_multiplexer::do_io_operation(io_multiplexer::io_operation_state *op) noexcept +inline void io_multiplexer::begin_io_operation(io_multiplexer::io_operation_state *op) noexcept { switch(op->state) { - case io_multiplexer::io_operation_state::state_t::read_requested: + case io_operation_state_type::read_requested: op->read_completed(op->payload.noncompleted.base ? op->h->_do_read(std::move(op->payload.noncompleted.base), std::move(op->payload.noncompleted.params.read.reqs), op->payload.noncompleted.d) : op->h->_do_read(std::move(op->payload.noncompleted.params.read.reqs), op->payload.noncompleted.d)); + op->read_finished(); break; - case io_multiplexer::io_operation_state::state_t::write_requested: + case io_operation_state_type::write_requested: op->write_completed(op->payload.noncompleted.base ? op->h->_do_write(std::move(op->payload.noncompleted.base), std::move(op->payload.noncompleted.params.write.reqs), op->payload.noncompleted.d) : op->h->_do_write(std::move(op->payload.noncompleted.params.write.reqs), op->payload.noncompleted.d)); + op->write_finished(); break; - case io_multiplexer::io_operation_state::state_t::barrier_requested: + case io_operation_state_type::barrier_requested: op->barrier_completed(op->h->_do_barrier(std::move(op->payload.noncompleted.params.barrier.reqs), op->payload.noncompleted.params.barrier.kind, op->payload.noncompleted.d)); + op->barrier_finished(); break; default: - // Invoke completion immediately with errc::invalid_argument - op->write_completed(errc::invalid_argument); - break; + abort(); } } diff --git a/include/llfio/v2.0/io_multiplexer.hpp b/include/llfio/v2.0/io_multiplexer.hpp index 7dc7750c..c33eea71 100644 --- a/include/llfio/v2.0/io_multiplexer.hpp +++ b/include/llfio/v2.0/io_multiplexer.hpp @@ -40,6 +40,106 @@ LLFIO_V2_NAMESPACE_EXPORT_BEGIN class io_handle; +//! The possible states of the i/o operation +enum class io_operation_state_type +{ + unknown, + read_requested, + read_initiated, + read_completed, + read_finished, + write_requested, + write_initiated, + barrier_requested, + barrier_initiated, + write_or_barrier_completed, + write_or_barrier_finished +}; +//! True if the i/o operation state is requested +constexpr inline bool is_requested(io_operation_state_type s) noexcept +{ + switch(s) + { + case io_operation_state_type::unknown: + case io_operation_state_type::read_initiated: + case io_operation_state_type::read_completed: + case io_operation_state_type::read_finished: + case io_operation_state_type::write_initiated: + case io_operation_state_type::write_or_barrier_completed: + case io_operation_state_type::write_or_barrier_finished: + case io_operation_state_type::barrier_initiated: + return false; + case io_operation_state_type::read_requested: + case io_operation_state_type::write_requested: + case io_operation_state_type::barrier_requested: + return true; + } + return false; +} +//! True if the i/o operation state is initiated +constexpr inline bool is_initiated(io_operation_state_type s) noexcept +{ + switch(s) + { + case io_operation_state_type::unknown: + case io_operation_state_type::read_requested: + case io_operation_state_type::read_completed: + case io_operation_state_type::read_finished: + case io_operation_state_type::write_requested: + case io_operation_state_type::barrier_requested: + case io_operation_state_type::write_or_barrier_completed: + case io_operation_state_type::write_or_barrier_finished: + return false; + case io_operation_state_type::read_initiated: + case io_operation_state_type::write_initiated: + case io_operation_state_type::barrier_initiated: + return true; + } + return false; +} +//! True if the i/o operation state is completed +constexpr inline bool is_completed(io_operation_state_type s) noexcept +{ + switch(s) + { + case io_operation_state_type::unknown: + case io_operation_state_type::read_requested: + case io_operation_state_type::read_initiated: + case io_operation_state_type::read_finished: + case io_operation_state_type::write_requested: + case io_operation_state_type::write_initiated: + case io_operation_state_type::barrier_requested: + case io_operation_state_type::barrier_initiated: + case io_operation_state_type::write_or_barrier_finished: + return false; + case io_operation_state_type::read_completed: + case io_operation_state_type::write_or_barrier_completed: + return true; + } + return false; +} +//! True if the i/o operation state is finished +constexpr inline bool is_finished(io_operation_state_type s) noexcept +{ + switch(s) + { + case io_operation_state_type::unknown: + case io_operation_state_type::read_requested: + case io_operation_state_type::read_initiated: + case io_operation_state_type::read_completed: + case io_operation_state_type::write_requested: + case io_operation_state_type::write_initiated: + case io_operation_state_type::barrier_requested: + case io_operation_state_type::barrier_initiated: + case io_operation_state_type::write_or_barrier_completed: + return false; + case io_operation_state_type::read_finished: + case io_operation_state_type::write_or_barrier_finished: + return true; + } + return false; +} + /*! \class io_multiplexer \brief A multiplexer of byte-orientated i/o. @@ -369,21 +469,34 @@ public: //! Implements `io_handle::allocate_registered_buffer()` LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<registered_buffer_type> do_io_handle_allocate_registered_buffer(io_handle *h, size_t &bytes) noexcept; - //! i/o operation state + /*! \brief A state for an i/o operation scheduled against an i/o multiplexer. + + The default implementation of this uses the blocking operations in `io_handle`, so setting + a default i/o multiplexer is the same as not setting an i/o multiplexer. + + The lifecycle for one of these is as follows: + + 1. i/o requested. This is after construction, before `io_multiplexer::begin_io_operation()` has been + called upon the i/o operation state. + + 2. i/o initiated. This state **may** be set by `io_multiplexer::begin_io_operation()` after the i/o has been + initiated with the OS by that function calling one of `.read_initiated()`, `.write_initiated()` or + `.barrier_initiated()`. If the i/o completed immediately in `io_multiplexer::begin_io_operation()`, + this state never occurs. + + 3. When the i/o completes, one of `.read_completed()`, `.write_completed()` or + `.barrier_completed()` will be called with the results of the i/o. These functions can be called + by **any** kernel thread, if the i/o multiplexer in use is used by multiple kernel threads. + The completion functions are *usually* invoked by somebody calling `io_multiplexer::check_io_operation()` + or `io_multiplexer::check_for_any_completed_io()`, but may also be called by an asynchronous system agent. + + 4. The i/o operation state may still be in use by others. You must not relocate in memory the + i/o operation state after `io_multiplexer::begin_io_operation()` returns until the `.finished()` + function is called. + */ struct io_operation_state { - enum class state_t - { - unknown, - read_requested, - read_initiated, - read_completed, - write_requested, - write_initiated, - barrier_requested, - barrier_initiated, - write_or_barrier_completed - } state{state_t::unknown}; + io_operation_state_type state{io_operation_state_type::unknown}; io_handle *h{nullptr}; union payload_t { _empty_t empty; @@ -471,97 +584,132 @@ public: constexpr io_operation_state() {} io_operation_state(io_handle *_h, registered_buffer_type &&b, deadline d, io_request<buffers_type> reqs) - : state(state_t::read_requested) + : state(io_operation_state_type::read_requested) , h(_h) , payload(std::move(b), d, std::move(reqs)) { } io_operation_state(io_handle *_h, registered_buffer_type &&b, deadline d, io_request<const_buffers_type> reqs) - : state(state_t::write_requested) + : state(io_operation_state_type::write_requested) , h(_h) , payload(std::move(b), d, std::move(reqs)) { } io_operation_state(io_handle *_h, registered_buffer_type &&b, deadline d, io_request<const_buffers_type> reqs, barrier_kind kind) - : state(state_t::barrier_requested) + : state(io_operation_state_type::barrier_requested) , h(_h) , payload(std::move(b), d, std::move(reqs), kind) { } - virtual ~io_operation_state() - { - clear_union_storage(); - } + virtual ~io_operation_state() { clear_union_storage(); } + //! Used to clear the union-stored state in this operation state void clear_union_storage() { switch(state) { - case state_t::unknown: + case io_operation_state_type::unknown: break; - case state_t::read_requested: - case state_t::read_initiated: + case io_operation_state_type::read_requested: + case io_operation_state_type::read_initiated: payload.noncompleted.base.~registered_buffer_type(); payload.noncompleted.d.~deadline(); payload.noncompleted.params.read.~read_params_t(); break; - case state_t::read_completed: + case io_operation_state_type::read_completed: + case io_operation_state_type::read_finished: payload.completed_read.~io_result<buffers_type>(); break; - case state_t::write_requested: - case state_t::write_initiated: + case io_operation_state_type::write_requested: + case io_operation_state_type::write_initiated: payload.noncompleted.base.~registered_buffer_type(); payload.noncompleted.d.~deadline(); payload.noncompleted.params.write.~write_params_t(); break; - case state_t::barrier_requested: - case state_t::barrier_initiated: + case io_operation_state_type::barrier_requested: + case io_operation_state_type::barrier_initiated: payload.noncompleted.base.~registered_buffer_type(); payload.noncompleted.d.~deadline(); payload.noncompleted.params.barrier.~barrier_params_t(); break; - case state_t::write_or_barrier_completed: + case io_operation_state_type::write_or_barrier_completed: + case io_operation_state_type::write_or_barrier_finished: payload.completed_write_or_barrier.~io_result<const_buffers_type>(); break; } - state = state_t::unknown; + state = io_operation_state_type::unknown; } //! Called when the read i/o has been initiated - virtual void read_initiated() { state = state_t::read_initiated; } + virtual void read_initiated() { state = io_operation_state_type::read_initiated; } //! Called when the read i/o has been completed virtual void read_completed(io_result<buffers_type> &&res) { clear_union_storage(); new(&payload.completed_read) io_result<buffers_type>(std::move(res)); - state = state_t::read_completed; + state = io_operation_state_type::read_completed; } + //! Called with the i/o operation state is no longer in used by others, and it can be recycled + virtual void read_finished() { state = io_operation_state_type::read_finished; } //! Called when the write i/o has been initiated - virtual void write_initiated() { state = state_t::write_initiated; } + virtual void write_initiated() { state = io_operation_state_type::write_initiated; } //! Called when the write i/o has been completed virtual void write_completed(io_result<const_buffers_type> &&res) { clear_union_storage(); new(&payload.completed_write_or_barrier) io_result<const_buffers_type>(std::move(res)); - state = state_t::write_or_barrier_completed; + state = io_operation_state_type::write_or_barrier_completed; } + //! Called with the i/o operation state is no longer in used by others, and it can be recycled + virtual void write_finished() { state = io_operation_state_type::write_or_barrier_finished; } //! Called when the barrier i/o has been initiated - virtual void barrier_initiated() { state = state_t::barrier_initiated; } + virtual void barrier_initiated() { state = io_operation_state_type::barrier_initiated; } //! Called when the barrier i/o has been completed virtual void barrier_completed(io_result<const_buffers_type> &&res) { clear_union_storage(); new(&payload.completed_write_or_barrier) io_result<const_buffers_type>(std::move(res)); - state = state_t::write_or_barrier_completed; + state = io_operation_state_type::write_or_barrier_completed; } + //! Called with the i/o operation state is no longer in used by others, and it can be recycled + virtual void barrier_finished() { state = io_operation_state_type::write_or_barrier_finished; } }; - //! Implements `io_handle::read()`, `io_handle::write()` and `io_handle::barrier()`. - LLFIO_HEADERS_ONLY_VIRTUAL_SPEC void do_io_operation(io_operation_state *op) noexcept; // default implementation is in io_handle.hpp + //! Begins a `io_handle::read()`, `io_handle::write()` and `io_handle::barrier()`. + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC void begin_io_operation(io_operation_state *op) noexcept; // default implementation is in io_handle.hpp + + //! Checks an initiated `io_handle::read()`, `io_handle::write()` and `io_handle::barrier()` for completion, returning true if it was completed + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC io_operation_state_type check_io_operation(io_operation_state *op) noexcept { return op->state; } + + //! Statistics about the just returned `wait_for_completed_io()` operation + struct wait_for_completed_io_statistics + { + size_t initiated_ios_completed{0}; //!< The number of initiated i/o which were completed by this call + }; + + /*! \brief Checks all i/o initiated on this i/o multiplexer to see which + have completed, trying without guarantee to complete no more than `max_completions` + completions. Can optionally sleep the thread until at least one initiated i/o completes, + though may return zero completed i/o's if another thread used `.wake_check_for_any_completed_io()`. + */ + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<wait_for_completed_io_statistics> check_for_any_completed_io(deadline d = std::chrono::seconds(0), size_t max_completions = (size_t) -1) noexcept + { + (void) d; + (void) max_completions; + return success(); + } + + /*! \brief Can be called from any thread to wake any other single thread + currently blocked within `check_for_any_completed_io()`. Which thread is + woken is not specified. + */ + LLFIO_HEADERS_ONLY_VIRTUAL_SPEC result<void> wake_check_for_any_completed_io() noexcept { return success(); } }; //! A unique ptr to an i/o multiplexer implementation. using io_multiplexer_ptr = std::unique_ptr<io_multiplexer>; -//! \brief Choose the best available i/o multiplexer implementation for this platform. -// LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_best_available(size_t threads) noexcept; +#if LLFIO_ENABLE_TEST_IO_MULTIPLEXERS +//! Namespace containing functions useful for test code +namespace test +{ #if defined(__linux__) || DOXYGEN_IS_IN_THE_HOUSE // LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_linux_epoll(size_t threads) noexcept; // LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_linux_io_uring() noexcept; @@ -570,8 +718,14 @@ using io_multiplexer_ptr = std::unique_ptr<io_multiplexer>; // LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_bsd_kqueue(size_t threads) noexcept; #endif #if defined(_WIN32) || DOXYGEN_IS_IN_THE_HOUSE -//! \brief Return an i/o multiplexer implemented using Microsoft Windows IOCPW -LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_win_iocp(size_t threads) noexcept; + /*! \brief Return a test i/o multiplexer implemented using Microsoft Windows IOCP. + + The multiplexer returned by this function is only a partial implementation, used + only by the test suite. + */ + LLFIO_HEADERS_ONLY_FUNC_SPEC result<io_multiplexer_ptr> multiplexer_win_iocp(size_t threads) noexcept; +#endif +} // namespace test #endif //! \brief Thread local settings |