From 3f1577fc4f5a3da414fae9621725352f01afc631 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 3 Nov 2026 04:14:37 +0000 Subject: [PATCH 1/3] refactor(io): encapsulate storage backend dispatch --- docs/architecture/README.md | 2 +- docs/architecture/io.md | 26 +++- docs/architecture/runtime.md | 5 +- include/bycorf/io/dpdk_backend.h | 3 +- include/bycorf/io/net_backend.h | 155 +++++++++------------- include/bycorf/io/spdk_storage.h | 5 +- include/bycorf/io/storage_backend.h | 189 +++++++++++++++++++++++++++ include/bycorf/runtime/worker.h | 128 ++++++------------ src/io/dpdk_backend.cpp | 16 +-- src/io/spdk_storage.cpp | 16 +-- src/runtime/worker.cpp | 72 +++++----- tests/io_uring_backpressure_test.cpp | 65 ++++++++- 12 files changed, 435 insertions(+), 247 deletions(-) create mode 100644 include/bycorf/io/storage_backend.h diff --git a/docs/architecture/README.md b/docs/architecture/README.md index ecf7049..a5ffa4c 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -23,7 +23,7 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging. | Module | Responsibility | Document | |---|---|---| | Runtime | Worker lifecycle, coroutine scheduling, cross-worker and foreign ingress | [Runtime](runtime.md) | -| I/O | io_uring, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | +| I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | | Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) | | RPC | Framed request/reply protocol above streams | [RPC](rpc.md) | diff --git a/docs/architecture/io.md b/docs/architecture/io.md index 227283e..c11d97e 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -32,11 +32,25 @@ allocation or runtime startup fails. This preserves DMA allocator identity, registered-resource ownership and native TCP state. Unsupported capabilities fail explicitly rather than falling back. -`NetBackend` owns the worker's single `IoUringBackend`. Its optional DPDK adapter -borrows this object; kernel sockets, storage, timers and MSG_RING use the same -submission/completion queues. Network-specific operations dispatch to the -selected backend. The worker drives completion polling and batching, and does -not create a second ring or worker pool for either accelerator. +`Worker` owns one `IoUringBackend`, borrowed by `NetBackend` and +`StorageBackend`. Kernel sockets, storage, timers and MSG_RING use the same +submission/completion queues. `NetBackend` selects kernel networking or its +optional DPDK adapter; `StorageBackend` selects kernel storage or its embedded +SPDK backend. SPDK default construction acquires no device resources; only the +selected backend is initialized. Storage dispatch uses its worker-local +initialized state, which remains active through shutdown completion draining. +Storage operations require a successfully initialized worker and stop before +its backend shutdown completes. The worker forwards storage operations without +selecting SPDK itself. Network event-loop polling and submission advance the +shared ring; separate storage polling advances only SPDK. Pending polled storage prevents +parking, while kernel storage uses the shared ring's wake/completion path. + +The worker initializes the ring before either adapter and stops storage, then +networking, before shutting down the ring. Kernel storage registrations live +until ring shutdown quiesces kernel access. Both adapters are destroyed before +the borrowed ring; neither creates a second ring or worker pool. SPDK callbacks +resolve their backend through the storage adapter using the owning worker's TLS, +including shutdown drains. Each io_uring backend owns a FIFO for accepted operations awaiting submission space. Successful admission does not mean the kernel has received the request. @@ -117,6 +131,6 @@ supports one runtime per process. Network I/O and descriptors stop on worker shutdown, but the complete BSD kernel subsystem teardown is not implemented. Sources: `include/bycorf/io/backend_options.h`, `src/io/backend_options.cpp`, -`include/bycorf/io/net_backend.h`, `src/io/io_uring_backend.cpp`, +`include/bycorf/io/net_backend.h`, `include/bycorf/io/storage_backend.h`, `src/io/io_uring_backend.cpp`, `src/io/storage.cpp`, `src/io/spdk_storage.cpp`, `src/io/dpdk_environment.cpp`, `cmake/Dpdk.cmake`, `cmake/Freebsd.cmake`, `cmake/build_freebsd.py`. diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index c0fbede..3df993d 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -24,8 +24,9 @@ inherited mask. Every thread completes affinity setup before any worker main function runs. An affinity or thread-launch failure prevents the worker main functions from entering partial service initialization. -A `Worker` owns its coroutine queues, connections, I/O backend, completion -handling, and optional storage poller. The scheduler bounds foreground and +A `Worker` owns its coroutine queues, connections, shared kernel I/O backend, +network and storage adapters, and completion handling. Both adapters borrow the +kernel backend; storage-specific polling is encapsulated by `StorageBackend`. The scheduler bounds foreground and background coroutine work and polls external completions between rounds. Workers submit queued foreground I/O and peer wakes before a maintenance slice, then submit any work queued by that slice before advancing to the next round. diff --git a/include/bycorf/io/dpdk_backend.h b/include/bycorf/io/dpdk_backend.h index 195ff5e..439a991 100644 --- a/include/bycorf/io/dpdk_backend.h +++ b/include/bycorf/io/dpdk_backend.h @@ -50,7 +50,8 @@ class DpdkBackend { // Release peers waiting for worker 0 if any worker fails during startup. static void AbortStartup(const absl::Status& reason); static void StopRuntime(); - absl::Status Init(const IoBackendOptions&, Worker*, int wake_fd); + // The worker initializes the borrowed ring before attaching networking. + absl::Status Init(const IoBackendOptions&, Worker*); void Shutdown(); bool Poll(); bool Wait(int timeout_ms); diff --git a/include/bycorf/io/net_backend.h b/include/bycorf/io/net_backend.h index 915798c..aa7264c 100644 --- a/include/bycorf/io/net_backend.h +++ b/include/bycorf/io/net_backend.h @@ -25,21 +25,36 @@ #endif namespace bycorf { -// One kernel ring per worker, shared by kernel sockets, storage, timers and -// wakeups. An optional DPDK adapter borrows that ring. Backend selection stays -// fixed after Init, so calls use a predictable branch without virtual dispatch. +// Routes networking through the worker-owned kernel ring or a DPDK adapter +// borrowing that same ring. Selection stays fixed after Init; the worker owns +// ring initialization and shutdown, including storage, timers and wakeups. class NetBackend { public: - absl::Status Init(const IoBackendOptions& options, Worker* worker, - int wake_fd) { + // The worker passes its ring directly so inlined kernel submissions do not + // load a stored ring pointer. DPDK borrows it for its adapter lifetime. + // Initialize networking after the worker initializes the shared ring. + absl::Status Init(IoUringBackend& kernel, const IoBackendOptions& options, + Worker* worker) { #if BYCORF_KERNEL_BYPASS if (DpdkNetworkEnabled()) { - dpdk_.emplace(kernel_); - return dpdk_->Init(options, worker, wake_fd); + dpdk_.emplace(kernel); + return dpdk_->Init(options, worker); } #endif - return kernel_.Init(options, worker, wake_fd); + (void)kernel; + (void)options; + (void)worker; + return absl::OkStatus(); } + // Release DPDK peers waiting for startup if shared-ring initialization fails. + void AbortStartup(const absl::Status& status) { +#if BYCORF_KERNEL_BYPASS + if (DpdkNetworkEnabled()) DpdkBackend::AbortStartup(status); +#else + (void)status; +#endif + } + // Stop networking while the borrowed ring is still alive. void Shutdown() { #if BYCORF_KERNEL_BYPASS if (dpdk_) { @@ -47,74 +62,79 @@ class NetBackend { return; } #endif - kernel_.Shutdown(); } - int WakeHandle() const noexcept { return kernel_.WakeHandle(); } - absl::Status SubmitSend(const RegisteredFile& file, + absl::Status SubmitSend(IoUringBackend& kernel, const RegisteredFile& file, std::span buffer, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitSend(file, buffer, tag); #endif - return kernel_.SubmitSend(file, buffer, tag); + return kernel.SubmitSend(file, buffer, tag); } - absl::Status SubmitSendMsg(const RegisteredFile& file, const msghdr* message, - IoCompletion* tag) { + absl::Status SubmitSendMsg(IoUringBackend& kernel, const RegisteredFile& file, + const msghdr* message, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitSendMsg(file, message, tag); #endif - return kernel_.SubmitSendMsg(file, message, tag); + return kernel.SubmitSendMsg(file, message, tag); } - absl::Status SubmitAcceptMultishot(int listen_fd, IoCompletion* tag) { + absl::Status SubmitAcceptMultishot(IoUringBackend& kernel, int listen_fd, + IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitAcceptMultishot(listen_fd, tag); #endif - return kernel_.SubmitAcceptMultishot(listen_fd, tag); + return kernel.SubmitAcceptMultishot(listen_fd, tag); } - absl::Status SubmitConnect(int fd, const sockaddr* address, - socklen_t address_length, IoCompletion* tag) { + absl::Status SubmitConnect(IoUringBackend& kernel, int fd, + const sockaddr* address, socklen_t address_length, + IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitConnect(fd, address, address_length, tag); #endif - return kernel_.SubmitConnect(fd, address, address_length, tag); + return kernel.SubmitConnect(fd, address, address_length, tag); } - absl::Status SubmitCancel(IoCompletion* target) { + absl::Status SubmitCancel(IoUringBackend& kernel, IoCompletion* target) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitCancel(target); #endif - return kernel_.SubmitCancel(target); + return kernel.SubmitCancel(target); } // Cancel kernel socket operations that have not acquired their fd reference // yet. Native DPDK handles must be excluded by the caller. - void CancelUnsubmittedForFd(int fd) noexcept { - kernel_.CancelUnsubmittedForFd(fd); + void CancelUnsubmittedForFd(IoUringBackend& kernel, int fd) noexcept { + kernel.CancelUnsubmittedForFd(fd); } - absl::Status StartRecvMultishot(Connection* connection) { + absl::Status StartRecvMultishot(IoUringBackend& kernel, + Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->StartRecvMultishot(connection); #endif - return kernel_.StartRecvMultishot(connection); + return kernel.StartRecvMultishot(connection); } // Null tag requests cancellation without a separate completion consumer. - absl::Status SubmitCancelRecv(Connection* connection, IoCompletion* tag) { + absl::Status SubmitCancelRecv(IoUringBackend& kernel, Connection* connection, + IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitCancelRecv(connection, tag); #endif - return kernel_.SubmitCancelRecv(connection, tag); + return kernel.SubmitCancelRecv(connection, tag); } - absl::Status StartPeerDisconnectPoll(Connection* connection) { + absl::Status StartPeerDisconnectPoll(IoUringBackend& kernel, + Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->StartPeerDisconnectPoll(connection); #endif - return kernel_.StartPeerDisconnectPoll(connection); + return kernel.StartPeerDisconnectPoll(connection); } - absl::Status CancelPeerDisconnectPoll(Connection* connection) { + absl::Status CancelPeerDisconnectPoll(IoUringBackend& kernel, + Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->CancelPeerDisconnectPoll(connection); #endif - return kernel_.CancelPeerDisconnectPoll(connection); + return kernel.CancelPeerDisconnectPoll(connection); } - std::span ViewRecvBuffer(const Connection* connection, + std::span ViewRecvBuffer(const IoUringBackend& kernel, + const Connection* connection, std::uint16_t buffer_id, std::size_t offset, std::size_t length) const { @@ -122,84 +142,35 @@ class NetBackend { if (dpdk_) return dpdk_->ViewRecvBuffer(connection, buffer_id, offset, length); #endif - return kernel_.ViewRecvBuffer(connection, buffer_id, offset, length); + return kernel.ViewRecvBuffer(connection, buffer_id, offset, length); } - void ReleaseRecvBuffer(Connection* connection, std::uint16_t buffer_id) { + void ReleaseRecvBuffer(IoUringBackend& kernel, Connection* connection, + std::uint16_t buffer_id) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->ReleaseRecvBuffer(connection, buffer_id); #endif - return kernel_.ReleaseRecvBuffer(connection, buffer_id); - } - absl::Status RegisterFixedFiles(unsigned count) { - return kernel_.RegisterFixedFiles(count); - } - absl::Status RegisterBuffers(std::span buffers) { - return kernel_.RegisterBuffers(buffers); - } - void UnregisterStorageResources() { - return kernel_.UnregisterStorageResources(); - } - absl::Status SubmitOpenDirect(std::string_view path, int flags, mode_t mode, - FixedFile file, IoCompletion* tag) { - return kernel_.SubmitOpenDirect(path, flags, mode, file, tag); - } - absl::Status SubmitCloseDirect(FixedFile file, IoCompletion* tag) { - return kernel_.SubmitCloseDirect(file, tag); + return kernel.ReleaseRecvBuffer(connection, buffer_id); } - absl::Status SubmitReadFixed(FixedFile file, FixedBuffer buffer, - std::uint64_t offset, IoCompletion* tag) { - return kernel_.SubmitReadFixed(file, buffer, offset, tag); - } - absl::Status SubmitRead(FixedFile file, std::span buffer, - std::uint64_t offset, IoCompletion* tag) { - return kernel_.SubmitRead(file, buffer, offset, tag); - } - absl::Status SubmitWrite(FixedFile file, std::span buffer, - std::uint64_t offset, IoCompletion* tag) { - return kernel_.SubmitWrite(file, buffer, offset, tag); - } - absl::Status SubmitWriteFixed(FixedFile file, FixedBuffer buffer, - std::uint64_t offset, IoCompletion* tag) { - return kernel_.SubmitWriteFixed(file, buffer, offset, tag); - } - absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag) { - return kernel_.SubmitFdatasync(file, tag); - } - absl::Status SubmitTimeout(const __kernel_timespec& timeout, - IoCompletion* tag) { - return kernel_.SubmitTimeout(timeout, tag); - } - absl::Status SubmitPoll(int fd, unsigned events, IoCompletion* tag) { - return kernel_.SubmitPoll(fd, events, tag); - } - absl::Status Submit() { + absl::Status Submit(IoUringBackend& kernel) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Submit(); #endif - return kernel_.Submit(); + return kernel.Submit(); } - bool Poll() { + bool Poll(IoUringBackend& kernel) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Poll(); #endif - return kernel_.Poll(); + return kernel.Poll(); } - // The shared kernel ring can retain work even when DPDK serves networking. - bool HasPending() const noexcept { return kernel_.HasPending(); } - bool Wait(int timeout_ms) { + bool Wait(IoUringBackend& kernel, int timeout_ms) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Wait(timeout_ms); #endif - return kernel_.Wait(timeout_ms); - } - void WakeRemote(int peer_ring_fd) noexcept { - return kernel_.WakeRemote(peer_ring_fd); + return kernel.Wait(timeout_ms); } - void WakeSelf() noexcept { return kernel_.WakeSelf(); } private: - // Declaration order ensures the borrowing adapter dies before its ring. - IoUringBackend kernel_; #if BYCORF_KERNEL_BYPASS // Inline the small adapter: packet/stream submissions already follow its // implementation pointer, so a separate heap adapter adds a dependent load diff --git a/include/bycorf/io/spdk_storage.h b/include/bycorf/io/spdk_storage.h index b0bea82..4824597 100644 --- a/include/bycorf/io/spdk_storage.h +++ b/include/bycorf/io/spdk_storage.h @@ -115,6 +115,9 @@ class SpdkStorageBackend { // Initialize only the storage backend embedded in worker. absl::Status Init(Worker* worker); void Shutdown(); + // Owner-thread state: true after successful Init until Shutdown finishes + // draining, including callbacks that submit more work during that drain. + bool IsInitialized() const noexcept { return worker_ != nullptr; } absl::Status RegisterFixedFiles(unsigned count); absl::Status RegisterBuffers(std::span buffers); @@ -182,6 +185,7 @@ class SpdkStorageBackend { OpenFile* Lookup(FixedFile file); static void CompleteAsync(void* context, const spdk_nvme_cpl* completion); + // Binding is also the initialized state; retain it through shutdown drains. Worker* worker_ = nullptr; std::vector files_; std::vector channels_; @@ -190,7 +194,6 @@ class SpdkStorageBackend { // Completion callbacks may open/close files and invalidate channel indices. std::size_t channel_generation_ = 0; std::size_t outstanding_ = 0; - bool initialized_ = false; }; #endif // BYCORF_KERNEL_BYPASS diff --git a/include/bycorf/io/storage_backend.h b/include/bycorf/io/storage_backend.h new file mode 100644 index 0000000..aaef38f --- /dev/null +++ b/include/bycorf/io/storage_backend.h @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BYCORF_IO_STORAGE_BACKEND_H_ +#define BYCORF_IO_STORAGE_BACKEND_H_ + +#include + +#include "bycorf/io/backend_options.h" +#include "bycorf/io/io_uring_backend.h" +#if BYCORF_KERNEL_BYPASS +#include "bycorf/io/spdk_storage.h" +#endif + +namespace bycorf { + +// Routes worker-owned storage operations using the frozen process selection. +// Borrows the worker's kernel backend; neither initializes nor shuts down its +// ring. The worker passes the ring directly to submissions, avoiding a stored +// pointer load, and keeps it alive until both adapters have stopped. +// Accepted operations retain caller-owned tags/buffers until completion. +class StorageBackend { + public: + struct PollResult { + bool did_work_ = false; + std::uint32_t completions_ = 0; + }; + + StorageBackend() = default; + StorageBackend(const StorageBackend&) = delete; + StorageBackend& operator=(const StorageBackend&) = delete; + + // Initialize after the shared ring, on the owning worker thread. + absl::Status Init(Worker* worker) { +#if BYCORF_KERNEL_BYPASS + if (SpdkStorageEnabled()) { + return spdk_.Init(worker); + } +#else + (void)worker; +#endif + return absl::OkStatus(); + } + // Drain the polled backend before the worker tears down the shared ring. + // Kernel storage resources retire with the ring, after all kernel I/O stops. + void Shutdown() { +#if BYCORF_KERNEL_BYPASS + spdk_.Shutdown(); +#endif + } + + // Kernel storage completions are driven by the shared ring's event loop. + // Only a backend without a kernel wake source needs separate polling and + // must keep the worker from parking while requests/completions are pending. + bool NeedsPolling() const noexcept { +#if BYCORF_KERNEL_BYPASS + return spdk_.IsInitialized() && spdk_.HasOutstanding(); +#else + return false; +#endif + } + // Call only after NeedsPolling returns true, on the owning worker. A zero + // limit preserves the backend's default completion budget. + PollResult Poll(unsigned max_completions = 0) { +#if BYCORF_KERNEL_BYPASS + // The worker calls Poll only after NeedsPolling, so no second selection + // check is needed on this hot path. + assert(spdk_.IsInitialized()); + const auto result = spdk_.Poll(max_completions); + return {result.did_work_, result.completions_}; +#else + (void)max_completions; +#endif + return {}; + } + + absl::Status RegisterFixedFiles(IoUringBackend& kernel, unsigned count) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.RegisterFixedFiles(count); + } +#endif + return kernel.RegisterFixedFiles(count); + } + absl::Status RegisterBuffers(IoUringBackend& kernel, + std::span buffers) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.RegisterBuffers(buffers); + } +#endif + return kernel.RegisterBuffers(buffers); + } + absl::Status SubmitOpenDirect(IoUringBackend& kernel, std::string_view path, + int flags, mode_t mode, FixedFile file, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitOpenDirect(path, flags, mode, file, tag); + } +#endif + return kernel.SubmitOpenDirect(path, flags, mode, file, tag); + } + absl::Status SubmitCloseDirect(IoUringBackend& kernel, FixedFile file, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitCloseDirect(file, tag); + } +#endif + return kernel.SubmitCloseDirect(file, tag); + } + absl::Status SubmitReadFixed(IoUringBackend& kernel, FixedFile file, + FixedBuffer buffer, std::uint64_t offset, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitReadFixed(file, buffer, offset, tag); + } +#endif + return kernel.SubmitReadFixed(file, buffer, offset, tag); + } + absl::Status SubmitRead(IoUringBackend& kernel, FixedFile file, + std::span buffer, std::uint64_t offset, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitRead(file, buffer, offset, tag); + } +#endif + return kernel.SubmitRead(file, buffer, offset, tag); + } + absl::Status SubmitWrite(IoUringBackend& kernel, FixedFile file, + std::span buffer, + std::uint64_t offset, IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitWrite(file, buffer, offset, tag); + } +#endif + return kernel.SubmitWrite(file, buffer, offset, tag); + } + absl::Status SubmitWriteFixed(IoUringBackend& kernel, FixedFile file, + FixedBuffer buffer, std::uint64_t offset, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitWriteFixed(file, buffer, offset, tag); + } +#endif + return kernel.SubmitWriteFixed(file, buffer, offset, tag); + } + absl::Status SubmitFdatasync(IoUringBackend& kernel, FixedFile file, + IoCompletion* tag) { +#if BYCORF_KERNEL_BYPASS + if (spdk_.IsInitialized()) { + return spdk_.SubmitFdatasync(file, tag); + } +#endif + return kernel.SubmitFdatasync(file, tag); + } + + private: +#if BYCORF_KERNEL_BYPASS + // Resolve callback ownership through worker TLS without adding an owner + // pointer to every I/O request. + friend class SpdkStorageBackend; + // Default construction acquires no device resources. Successful Init marks + // this backend active; reuse that state instead of a second selection flag. + // Storage submissions require a successfully initialized, live worker. + SpdkStorageBackend spdk_; +#endif +}; +} // namespace bycorf + +#endif diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index af4d851..fadcbb5 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -34,8 +34,8 @@ #include "bycorf/base/inline_ring_queue.h" #include "bycorf/io/completion.h" #include "bycorf/io/net_backend.h" -#include "bycorf/io/spdk_storage.h" #include "bycorf/io/storage.h" +#include "bycorf/io/storage_backend.h" #include "bycorf/net/connection.h" #include "bycorf/runtime/cross_core.h" #include "bycorf/runtime/task.h" @@ -229,124 +229,95 @@ class Worker { absl::Status SubmitSend(const RegisteredFile& file, std::span buffer, IoCompletion* tag) { - return backend_.SubmitSend(file, buffer, tag); + return networking_backend_.SubmitSend(kernel_backend_, file, buffer, tag); } absl::Status SubmitSendMsg(const RegisteredFile& file, const msghdr* message, IoCompletion* tag) { - return backend_.SubmitSendMsg(file, message, tag); + return networking_backend_.SubmitSendMsg(kernel_backend_, file, message, + tag); } absl::Status SubmitAcceptMultishot(int listen_fd, IoCompletion* tag) { - return backend_.SubmitAcceptMultishot(listen_fd, tag); + return networking_backend_.SubmitAcceptMultishot(kernel_backend_, listen_fd, + tag); } // Async connect on a raw fd (pre-registration); see ConnectTcp. absl::Status SubmitConnect(int fd, const sockaddr* address, socklen_t address_length, IoCompletion* tag) { - return backend_.SubmitConnect(fd, address, address_length, tag); + return networking_backend_.SubmitConnect(kernel_backend_, fd, address, + address_length, tag); } // Best-effort cancel of the in-flight SQE tagged `target`; owner-thread // only (the ring is single-issuer). absl::Status SubmitCancel(IoCompletion* target) { - return backend_.SubmitCancel(target); + return networking_backend_.SubmitCancel(kernel_backend_, target); } // Before closing a kernel socket, retire accepted operations whose fd has // not yet been consumed by the kernel. Completions remain deferred. void CancelUnsubmittedForFd(int fd) noexcept { - backend_.CancelUnsubmittedForFd(fd); + networking_backend_.CancelUnsubmittedForFd(kernel_backend_, fd); } absl::Status EnsureRecvArmed(Connection* connection) { - return backend_.StartRecvMultishot(connection); + return networking_backend_.StartRecvMultishot(kernel_backend_, connection); } absl::Status SubmitCancelRecv(Connection* connection, IoCompletion* tag) { - return backend_.SubmitCancelRecv(connection, tag); + return networking_backend_.SubmitCancelRecv(kernel_backend_, connection, + tag); } absl::Status EnsurePeerDisconnectPollArmed(Connection* connection) { - return backend_.StartPeerDisconnectPoll(connection); + return networking_backend_.StartPeerDisconnectPoll(kernel_backend_, + connection); } absl::Status CancelPeerDisconnectPoll(Connection* connection) { - return backend_.CancelPeerDisconnectPoll(connection); + return networking_backend_.CancelPeerDisconnectPoll(kernel_backend_, + connection); } std::span ViewMultishotBuffer(const Connection* connection, std::uint16_t buffer_id, std::size_t offset, std::size_t length) const { - return backend_.ViewRecvBuffer(connection, buffer_id, offset, length); + return networking_backend_.ViewRecvBuffer(kernel_backend_, connection, + buffer_id, offset, length); } void ReleaseReceivedBuffer(Connection* connection, std::uint16_t buffer_id) { - backend_.ReleaseRecvBuffer(connection, buffer_id); + networking_backend_.ReleaseRecvBuffer(kernel_backend_, connection, + buffer_id); } absl::Status RegisterFixedFiles(unsigned count) { - absl::Status status; -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - status = storage_backend_.RegisterFixedFiles(count); - } else -#endif - { - status = backend_.RegisterFixedFiles(count); - } + auto status = storage_backend_.RegisterFixedFiles(kernel_backend_, count); if (status.ok()) { storage_file_io_stats_.assign(count, StorageFileIoStats{}); } return status; } absl::Status RegisterBuffers(std::span buffers) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.RegisterBuffers(buffers); - } -#endif - return backend_.RegisterBuffers(buffers); + return storage_backend_.RegisterBuffers(kernel_backend_, buffers); } // Storage submissions retain the tag and referenced caller-owned buffers // until completion, including while queued. Each tag may have only one // unfinished storage operation; its completion may submit the next one. absl::Status SubmitOpenDirect(std::string_view path, int flags, mode_t mode, FixedFile file, IoCompletion* tag) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.SubmitOpenDirect(path, flags, mode, file, tag); - } -#endif - return backend_.SubmitOpenDirect(path, flags, mode, file, tag); + return storage_backend_.SubmitOpenDirect(kernel_backend_, path, flags, mode, + file, tag); } absl::Status SubmitCloseDirect(FixedFile file, IoCompletion* tag) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.SubmitCloseDirect(file, tag); - } -#endif - return backend_.SubmitCloseDirect(file, tag); + return storage_backend_.SubmitCloseDirect(kernel_backend_, file, tag); } absl::Status SubmitReadFixed(FixedFile file, FixedBuffer buffer, std::uint64_t offset, IoCompletion* tag) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.SubmitReadFixed(file, buffer, offset, tag); - } -#endif - return backend_.SubmitReadFixed(file, buffer, offset, tag); + return storage_backend_.SubmitReadFixed(kernel_backend_, file, buffer, + offset, tag); } absl::Status SubmitRead(FixedFile file, std::span buffer, std::uint64_t offset, IoCompletion* tag) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.SubmitRead(file, buffer, offset, tag); - } -#endif - return backend_.SubmitRead(file, buffer, offset, tag); + return storage_backend_.SubmitRead(kernel_backend_, file, buffer, offset, + tag); } absl::Status SubmitWrite(FixedFile file, std::span buffer, std::uint64_t offset, IoCompletion* tag) { - absl::Status status; -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - status = storage_backend_.SubmitWrite(file, buffer, offset, tag); - } else -#endif - { - status = backend_.SubmitWrite(file, buffer, offset, tag); - } + auto status = storage_backend_.SubmitWrite(kernel_backend_, file, buffer, + offset, tag); if (status.ok() && file.index_ < storage_file_io_stats_.size()) { storage_file_io_stats_[file.index_].submitted_write_bytes_ += buffer.size(); @@ -355,15 +326,8 @@ class Worker { } absl::Status SubmitWriteFixed(FixedFile file, FixedBuffer buffer, std::uint64_t offset, IoCompletion* tag) { - absl::Status status; -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - status = storage_backend_.SubmitWriteFixed(file, buffer, offset, tag); - } else -#endif - { - status = backend_.SubmitWriteFixed(file, buffer, offset, tag); - } + auto status = storage_backend_.SubmitWriteFixed(kernel_backend_, file, + buffer, offset, tag); if (status.ok() && file.index_ < storage_file_io_stats_.size()) { storage_file_io_stats_[file.index_].submitted_write_bytes_ += buffer.size_; @@ -371,16 +335,11 @@ class Worker { return status; } absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag) { -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) { - return storage_backend_.SubmitFdatasync(file, tag); - } -#endif - return backend_.SubmitFdatasync(file, tag); + return storage_backend_.SubmitFdatasync(kernel_backend_, file, tag); } absl::Status SubmitTimeout(const __kernel_timespec& timeout, IoCompletion* tag) { - return backend_.SubmitTimeout(timeout, tag); + return kernel_backend_.SubmitTimeout(timeout, tag); } // Takes ownership of a connection with a valid socket; returns nullptr for @@ -420,8 +379,7 @@ class Worker { }; #if BYCORF_KERNEL_BYPASS - // SPDK callbacks resolve this worker's sole storage backend through the - // existing worker TLS instead of storing an owner pointer in every request. + // Completion callbacks locate the embedded SPDK instance through worker TLS. friend class SpdkStorageBackend; #endif @@ -438,13 +396,11 @@ class Worker { bool ShouldRunBackground() const noexcept; void MaybeResetRuntimeWindow() noexcept; void RecordRound(std::int64_t start_cycles) noexcept; - void Flush(); // FlushWakes() + backend_.Submit() + void Flush(); // FlushWakes() + networking_backend_.Submit(kernel_backend_) void FlushWakes(); // wake every marked, parked target once void CheckIdleConnections(); bool BusyPoll(); -#if BYCORF_KERNEL_BYPASS bool PollStorage(); -#endif bool CanReclaim(const Connection& connection) const noexcept; void ReclaimConnections(); void DiscardReceivedBuffers(Connection* connection); @@ -453,10 +409,10 @@ class Worker { bool DrainCrossCoreOverflow(); bool DrainCrossCore(); - NetBackend backend_{}; -#if BYCORF_KERNEL_BYPASS - SpdkStorageBackend storage_backend_{}; -#endif + // Declare the shared ring first so both borrowing adapters die before it. + IoUringBackend kernel_backend_{}; + NetBackend networking_backend_{}; + StorageBackend storage_backend_{}; bool initialized_ = false; // A fatal enter failure must leave RunOnce through the existing backend // failure path, rather than spinning forever on accepted pending work. diff --git a/src/io/dpdk_backend.cpp b/src/io/dpdk_backend.cpp index 0044676..10b0c44 100644 --- a/src/io/dpdk_backend.cpp +++ b/src/io/dpdk_backend.cpp @@ -859,19 +859,12 @@ void DpdkBackend::AbortStartup(const absl::Status& reason) { if (fabric.failure.ok()) fabric.failure = reason; fabric.attached.notify_all(); } -absl::Status DpdkBackend::Init(const IoBackendOptions& options, Worker* worker, - int wake_fd) { +absl::Status DpdkBackend::Init(const IoBackendOptions& options, + Worker* worker) { if (impl_->initialized) return absl::OkStatus(); if (!fabric.prepared || fabric.stopped) return absl::FailedPreconditionError( "DPDK Runtime must prepare before worker initialization"); - auto kernel_options = options; - kernel_options.recv_buffer_count_ = 0; - auto status = kernel_.Init(kernel_options, worker, wake_fd); - if (!status.ok()) { - AbortStartup(status); - return status; - } impl_->worker = worker; impl_->id = worker->id(); impl_->buffer_size = options.recv_buffer_size_; @@ -880,9 +873,8 @@ absl::Status DpdkBackend::Init(const IoBackendOptions& options, Worker* worker, impl_->next_handle = kFirstHandle | (impl_->id << kHandleBits); current_ = impl_.get(); if (rte_thread_register()) { - status = DeviceError("register worker with EAL", rte_errno); + auto status = DeviceError("register worker with EAL", rte_errno); AbortStartup(status); - kernel_.Shutdown(); current_ = nullptr; return status; } @@ -912,7 +904,6 @@ absl::Status DpdkBackend::Init(const IoBackendOptions& options, Worker* worker, fabric.attached.notify_all(); rte_thread_unregister(); current_ = nullptr; - kernel_.Shutdown(); return fabric.failure; } fabric.lanes[id].ring_fd = kernel_.WakeHandle(); @@ -943,7 +934,6 @@ void DpdkBackend::Shutdown() { (unsigned long long)impl_->forwarded_tx, (unsigned long long)impl_->drops, (unsigned long long)impl_->waits, (unsigned long long)impl_->arms, (unsigned long long)impl_->notifications); - kernel_.Shutdown(); rte_thread_unregister(); current_ = nullptr; impl_->initialized = false; diff --git a/src/io/spdk_storage.cpp b/src/io/spdk_storage.cpp index 3a5629e..c89660d 100644 --- a/src/io/spdk_storage.cpp +++ b/src/io/spdk_storage.cpp @@ -413,7 +413,7 @@ void SpdkStorageBackend::CompleteAsync(void* context, spdk_nvme_cpl_is_error(completion) ? -EIO : tag->success_result_; Worker* worker = ThisWorker().self_; assert(worker != nullptr); - SpdkStorageBackend* backend = &worker->storage_backend_; + SpdkStorageBackend* backend = &worker->storage_backend_.spdk_; // Complete may resubmit this tag or destroy it. Read its result first and // never access caller-owned state after dispatching the terminal completion. backend->CompleteRequest(tag, result); @@ -428,17 +428,16 @@ void SpdkStorageBackend::CompleteRequest(IoCompletion* tag, int result) { } absl::Status SpdkStorageBackend::Init(Worker* worker) { - if (worker == nullptr || &worker->storage_backend_ != this) { + if (worker == nullptr || &worker->storage_backend_.spdk_ != this) { return absl::Status(absl::StatusCode::kInvalidArgument, "SPDK storage backend must be owned by its worker"); } worker_ = worker; - initialized_ = true; return absl::OkStatus(); } void SpdkStorageBackend::Shutdown() { - if (!initialized_) { + if (worker_ == nullptr) { return; } while (HasOutstanding()) { @@ -455,12 +454,13 @@ void SpdkStorageBackend::Shutdown() { pending_completions_.clear(); next_poll_channel_ = 0; channel_generation_ = 0; + // Keep dispatch on SPDK until every completion and any resubmission from + // its callback has drained; only then retire this backend. worker_ = nullptr; - initialized_ = false; } absl::Status SpdkStorageBackend::RegisterFixedFiles(unsigned count) { - if (!initialized_ || count == 0 || !files_.empty()) { + if (worker_ == nullptr || count == 0 || !files_.empty()) { return absl::Status(absl::StatusCode::kFailedPrecondition, "invalid SPDK fixed-file table registration"); } @@ -470,7 +470,7 @@ absl::Status SpdkStorageBackend::RegisterFixedFiles(unsigned count) { absl::Status SpdkStorageBackend::RegisterBuffers( std::span buffers) { - if (!initialized_ || buffers.empty()) { + if (worker_ == nullptr || buffers.empty()) { return absl::Status(absl::StatusCode::kFailedPrecondition, "invalid SPDK DMA-buffer registration"); } @@ -738,7 +738,7 @@ ABSL_ATTRIBUTE_NOINLINE void SpdkStorageBackend::RetryDeferred( } SpdkPollResult SpdkStorageBackend::Poll(unsigned max_completions) { - assert(!initialized_ || ThisWorker().self_ == worker_); + assert(worker_ == nullptr || ThisWorker().self_ == worker_); SpdkPollResult result; if (!pending_completions_.empty()) { std::vector> completions; diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index fe2341e..2535d7c 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -205,22 +205,30 @@ absl::Status Worker::Init(const WorkerOptions& options) { IoBackendOptions backend_options; backend_options.ring_entries_ = options.ring_entries_; backend_options.recv_buffer_count_ = options.recv_buffer_count_; - // Runtime pre-created the eventfd; the backend reuses it (does not own it). - auto status = - backend_.Init(backend_options, this, cross_core_->mailbox(id_).wake_fd_); + // DPDK receives use adapter-owned buffers instead of a kernel receive ring. + auto kernel_options = backend_options; + if (DpdkNetworkEnabled()) kernel_options.recv_buffer_count_ = 0; + // Runtime pre-created the eventfd; the kernel backend borrows it. + auto status = kernel_backend_.Init(kernel_options, this, + cross_core_->mailbox(id_).wake_fd_); if (!status.ok()) { + networking_backend_.AbortStartup(status); return status; } -#if BYCORF_KERNEL_BYPASS - if (SpdkStorageEnabled()) status = storage_backend_.Init(this); + status = networking_backend_.Init(kernel_backend_, backend_options, this); if (!status.ok()) { - backend_.Shutdown(); + kernel_backend_.Shutdown(); + return status; + } + status = storage_backend_.Init(this); + if (!status.ok()) { + networking_backend_.Shutdown(); + kernel_backend_.Shutdown(); return status; } -#endif // Advertise our ring so other workers can wake us via MSG_RING. - cross_core_->mailbox(id_).ring_fd_ = backend_.WakeHandle(); + cross_core_->mailbox(id_).ring_fd_ = kernel_backend_.WakeHandle(); submit_failed_ = false; initialized_ = true; return absl::OkStatus(); @@ -230,10 +238,9 @@ void Worker::Shutdown() { if (!initialized_) { return; } -#if BYCORF_KERNEL_BYPASS storage_backend_.Shutdown(); -#endif - backend_.Shutdown(); + networking_backend_.Shutdown(); + kernel_backend_.Shutdown(); initialized_ = false; } @@ -361,7 +368,8 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, // Stop the independent peer-disconnect observer before retiring the // Connection storage. Its completion owns one inflight operation and will // make the connection reclaimable. - (void)backend_.CancelPeerDisconnectPoll(connection); + (void)networking_backend_.CancelPeerDisconnectPoll(kernel_backend_, + connection); // close(fd) alone does not release a socket held by an io_uring receive. // Cancel by operation identity before retiring it so the peer observes the @@ -369,7 +377,8 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, // Unlike shutdown(), this preserves a duplicated socket after PauseRead() // hands it to another worker. DPDK drains its corresponding receive here. if (connection->recv_armed_) { - (void)backend_.SubmitCancelRecv(connection, nullptr); + (void)networking_backend_.SubmitCancelRecv(kernel_backend_, connection, + nullptr); } if (connection->file_.fd_ >= 0) { @@ -377,7 +386,7 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, if (!detail::IsDpdkSocket(fd)) { // An accepted operation may still live in the software queue or SQ. // Retire it before close lets another socket reuse this descriptor. - backend_.CancelUnsubmittedForFd(fd); + networking_backend_.CancelUnsubmittedForFd(kernel_backend_, fd); } connection->file_.fd_ = -1; connection->closed_ = true; @@ -795,7 +804,8 @@ void Worker::DiscardReceivedBuffers(Connection* connection) { // ReleaseRecvBuffer only recycles backend storage; it does not mutate this // queue or resume readers. Clear once to avoid shrinking while discarding. for (std::uint32_t i = 0; i < count; ++i) { - backend_.ReleaseRecvBuffer(connection, received[i].buffer_id_); + networking_backend_.ReleaseRecvBuffer(kernel_backend_, connection, + received[i].buffer_id_); } received.clear(); } @@ -1044,7 +1054,7 @@ void Worker::FlushWakes() { #endif if (parked) { ++wake_sent_; - backend_.WakeRemote(mb.ring_fd_); + kernel_backend_.WakeRemote(mb.ring_fd_); } } w.wake_list_.clear(); @@ -1058,10 +1068,8 @@ bool Worker::BusyPoll() { CycleNow() + static_cast(busy_poll_cycles_); do { bool did_work = DrainCrossCore(); - did_work |= backend_.Poll(); -#if BYCORF_KERNEL_BYPASS + did_work |= networking_backend_.Poll(kernel_backend_); did_work |= PollStorage(); -#endif if (did_work) { const std::int64_t round_start = CycleNow(); MergeDeferred(); @@ -1095,15 +1103,14 @@ bool Worker::BusyPoll() { return false; } -#if BYCORF_KERNEL_BYPASS bool Worker::PollStorage() { // Submissions and completions are worker-owned. With neither device I/O // nor deferred open/close completions pending, polling cannot make progress. // Use the same predicate as the idle/park decision so queued completions // still run even when the hardware outstanding count is zero. - if (!SpdkStorageEnabled() || !storage_backend_.HasOutstanding()) return false; + if (!storage_backend_.NeedsPolling()) return false; const std::int64_t start = CycleNow(); - const SpdkPollResult result = + const auto result = storage_backend_.Poll(options_.spdk_max_completions_per_poll_); const std::uint64_t elapsed = static_cast(CycleNow() - start); ++scheduler_stats_.storage_poll_calls_; @@ -1116,11 +1123,10 @@ bool Worker::PollStorage() { std::max(scheduler_stats_.storage_max_poll_cycles_, elapsed); return result.did_work_; } -#endif void Worker::Flush() { FlushWakes(); - const auto status = backend_.Submit(); + const auto status = networking_backend_.Submit(kernel_backend_); if (!status.ok()) [[unlikely]] { // CQ backpressure and interrupted enters are retried by the next round. // A permanent error cannot make progress by waiting for queue capacity. @@ -1145,12 +1151,11 @@ bool Worker::RunOnce(bool wait_for_completion) { !foreground_remote_work_.empty() || !background_ready_.empty() || !next_background_ready_.empty() || !background_remote_work_.empty(); did_work |= DrainCrossCore(); - did_work |= backend_.Poll(); -#if BYCORF_KERNEL_BYPASS + did_work |= networking_backend_.Poll(kernel_backend_); // This extra foreground slice exists to get ahead of storage completions. // Without pending storage work, the normal slice below already serves the // same queues; splitting it adds a second deadline and accounting pass. - if (SpdkStorageEnabled() && storage_backend_.HasOutstanding() && + if (storage_backend_.NeedsPolling() && options_.spdk_foreground_pre_poll_us_ != 0 && (!ready_.empty() || !next_ready_.empty() || !foreground_remote_work_.empty())) { @@ -1173,7 +1178,6 @@ bool Worker::RunOnce(bool wait_for_completion) { } } did_work |= PollStorage(); -#endif MergeDeferred(); const std::int64_t foreground_start = CycleNow(); @@ -1227,7 +1231,7 @@ bool Worker::RunOnce(bool wait_for_completion) { if (submit_failed_) return false; // Submission can free SQ slots without producing a completion. Keep both // normal rounds and shutdown drains running until deferred work advances. - did_work |= backend_.HasPending(); + did_work |= kernel_backend_.HasPending(); ReclaimConnections(); MaybeResetRuntimeWindow(); RecordRound(round_start); @@ -1243,13 +1247,10 @@ bool Worker::RunOnce(bool wait_for_completion) { return !submit_failed_; } -#if BYCORF_KERNEL_BYPASS - // SPDK is a polled-mode driver and has no fd that can wake io_uring. Keep - // driving its completion queues while device I/O is outstanding. - if (storage_backend_.HasOutstanding()) { + // A polled storage backend has no kernel wake source for its completions. + if (storage_backend_.NeedsPolling()) { return true; } -#endif // 4. Idle — park. Check idle timeouts only here, off the hot path. The // wake_seq @@ -1272,7 +1273,8 @@ bool Worker::RunOnce(bool wait_for_completion) { } const int timeout_ms = options_.idle_timeout_ms_ > 0 ? 100 : -1; - const bool ok = backend_.Wait(timeout_ms); // blocks; dispatches on wake + const bool ok = networking_backend_.Wait( + kernel_backend_, timeout_ms); // blocks; dispatches on wake mb.wake_seq_.store(0, std::memory_order_release); // leave the parked state return ok; // next iteration drains what Wait dispatched } diff --git a/tests/io_uring_backpressure_test.cpp b/tests/io_uring_backpressure_test.cpp index de96c82..46413a6 100644 --- a/tests/io_uring_backpressure_test.cpp +++ b/tests/io_uring_backpressure_test.cpp @@ -56,8 +56,8 @@ struct Fixture { bycorf::IoUringBackend backend; }; -template -void Drain(bycorf::IoUringBackend& backend, Predicate done) { +template +void Drain(Backend& backend, Predicate done) { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); while (!done()) { @@ -400,4 +400,65 @@ TEST(IoUringBackpressure, StorageReads) { std::cout << "PASS queued fixed-file reads preserve buffers and offsets\n"; } +// Both facades must submit to the same ring: the network event loop also +// delivers storage completions, and stopping either facade must leave the +// worker-owned ring available until the worker shuts it down. +TEST(IoUringBackpressure, NetworkAndStorageShareRing) { + Fixture fixture; + ASSERT_NO_FATAL_FAILURE(fixture.Init()); + struct NetworkLoop { + bycorf::NetBackend network; + bycorf::IoUringBackend& kernel; + absl::Status Submit() { return network.Submit(kernel); } + bool Poll() { return network.Poll(kernel); } + } loop{{}, fixture.backend}; + auto& network = loop.network; + bycorf::StorageBackend storage; + ASSERT_TRUE(network.Init(fixture.backend, {}, &fixture.worker).ok()); + ASSERT_TRUE(storage.Init(&fixture.worker).ok()); + ASSERT_FALSE(storage.NeedsPolling()); + ASSERT_TRUE(storage.RegisterFixedFiles(fixture.backend, 1).ok()); + + Completion opened; + ASSERT_TRUE(storage + .SubmitOpenDirect(fixture.backend, "/dev/zero", O_RDONLY, 0, + {0}, &opened) + .ok()); + Drain(loop, [&] { return opened.calls != 0; }); + ASSERT_EQ(opened.result, 0); + + int sockets[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0, + sockets), + 0); + std::array bytes; + bytes.fill(std::byte{1}); + const std::array payload{std::byte{42}}; + Completion read, sent; + ASSERT_TRUE(storage.SubmitRead(fixture.backend, {0}, bytes, 0, &read).ok()); + ASSERT_TRUE( + network.SubmitSend(fixture.backend, {.fd_ = sockets[0]}, payload, &sent) + .ok()); + ASSERT_EQ(read.calls, 0u); + ASSERT_EQ(sent.calls, 0u); + // Kernel storage never requests a separate poll or keeps the worker spinning. + ASSERT_FALSE(storage.NeedsPolling()); + Drain(loop, [&] { return read.calls != 0 && sent.calls != 0; }); + ASSERT_EQ(read.result, bytes.size()); + ASSERT_EQ(sent.result, 1); + EXPECT_EQ(bytes, (std::array{})); + std::byte received; + ASSERT_EQ(recv(sockets[1], &received, 1, 0), 1); + EXPECT_EQ(received, payload[0]); + ASSERT_EQ(close(sockets[0]), 0); + ASSERT_EQ(close(sockets[1]), 0); + + storage.Shutdown(); + network.Shutdown(); + Completion closed; + ASSERT_TRUE(fixture.backend.SubmitCloseDirect({0}, &closed).ok()); + Drain(fixture.backend, [&] { return closed.calls != 0; }); + ASSERT_EQ(closed.result, 0); +} + } // namespace From 138a5222a02c8fef8d639d4f7febbda3f1ca478c Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 3 Nov 2026 04:19:56 +0000 Subject: [PATCH 2/3] refactor(io): rename SPDK backend to SpdkBackend --- include/bycorf/io/completion.h | 2 +- include/bycorf/io/spdk_storage.h | 10 ++-- include/bycorf/io/storage_backend.h | 4 +- include/bycorf/runtime/worker.h | 2 +- src/io/spdk_storage.cpp | 85 +++++++++++++---------------- 5 files changed, 47 insertions(+), 56 deletions(-) diff --git a/include/bycorf/io/completion.h b/include/bycorf/io/completion.h index 0a2f91d..372e93c 100644 --- a/include/bycorf/io/completion.h +++ b/include/bycorf/io/completion.h @@ -48,7 +48,7 @@ class IoCompletion { std::coroutine_handle<> awaiting_{}; private: - friend class SpdkStorageBackend; + friend class SpdkBackend; // Status-only storage completions need the expected successful byte count // (zero for flush). Keep it with the caller-owned tag so submitted commands diff --git a/include/bycorf/io/spdk_storage.h b/include/bycorf/io/spdk_storage.h index 4824597..9fdbed5 100644 --- a/include/bycorf/io/spdk_storage.h +++ b/include/bycorf/io/spdk_storage.h @@ -105,12 +105,12 @@ void FreeStorageBuffer(void* buffer, std::size_t alignment) noexcept; // capacity. The backend owns waiting request metadata; each tag represents one // unfinished request. Submit methods never dispatch completions inline. Poll // drives both device completions and deferred submissions. -class SpdkStorageBackend { +class SpdkBackend { public: - SpdkStorageBackend() = default; - SpdkStorageBackend(const SpdkStorageBackend&) = delete; - SpdkStorageBackend& operator=(const SpdkStorageBackend&) = delete; - ~SpdkStorageBackend(); + SpdkBackend() = default; + SpdkBackend(const SpdkBackend&) = delete; + SpdkBackend& operator=(const SpdkBackend&) = delete; + ~SpdkBackend(); // Initialize only the storage backend embedded in worker. absl::Status Init(Worker* worker); diff --git a/include/bycorf/io/storage_backend.h b/include/bycorf/io/storage_backend.h index aaef38f..161c00b 100644 --- a/include/bycorf/io/storage_backend.h +++ b/include/bycorf/io/storage_backend.h @@ -177,11 +177,11 @@ class StorageBackend { #if BYCORF_KERNEL_BYPASS // Resolve callback ownership through worker TLS without adding an owner // pointer to every I/O request. - friend class SpdkStorageBackend; + friend class SpdkBackend; // Default construction acquires no device resources. Successful Init marks // this backend active; reuse that state instead of a second selection flag. // Storage submissions require a successfully initialized, live worker. - SpdkStorageBackend spdk_; + SpdkBackend spdk_; #endif }; } // namespace bycorf diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index fadcbb5..d274379 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -380,7 +380,7 @@ class Worker { #if BYCORF_KERNEL_BYPASS // Completion callbacks locate the embedded SPDK instance through worker TLS. - friend class SpdkStorageBackend; + friend class SpdkBackend; #endif struct StorageFileIoStats { diff --git a/src/io/spdk_storage.cpp b/src/io/spdk_storage.cpp index c89660d..d300541 100644 --- a/src/io/spdk_storage.cpp +++ b/src/io/spdk_storage.cpp @@ -406,28 +406,28 @@ void FreeStorageBuffer(void* buffer, std::size_t alignment) noexcept { ::operator delete[](buffer, std::align_val_t(alignment)); } -void SpdkStorageBackend::CompleteAsync(void* context, - const spdk_nvme_cpl* completion) { +void SpdkBackend::CompleteAsync(void* context, + const spdk_nvme_cpl* completion) { auto* tag = static_cast(context); const int result = spdk_nvme_cpl_is_error(completion) ? -EIO : tag->success_result_; Worker* worker = ThisWorker().self_; assert(worker != nullptr); - SpdkStorageBackend* backend = &worker->storage_backend_.spdk_; + SpdkBackend* backend = &worker->storage_backend_.spdk_; // Complete may resubmit this tag or destroy it. Read its result first and // never access caller-owned state after dispatching the terminal completion. backend->CompleteRequest(tag, result); } -SpdkStorageBackend::~SpdkStorageBackend() { Shutdown(); } +SpdkBackend::~SpdkBackend() { Shutdown(); } -void SpdkStorageBackend::CompleteRequest(IoCompletion* tag, int result) { +void SpdkBackend::CompleteRequest(IoCompletion* tag, int result) { assert(outstanding_ != 0); --outstanding_; tag->Complete(*worker_, result, 0); } -absl::Status SpdkStorageBackend::Init(Worker* worker) { +absl::Status SpdkBackend::Init(Worker* worker) { if (worker == nullptr || &worker->storage_backend_.spdk_ != this) { return absl::Status(absl::StatusCode::kInvalidArgument, "SPDK storage backend must be owned by its worker"); @@ -436,7 +436,7 @@ absl::Status SpdkStorageBackend::Init(Worker* worker) { return absl::OkStatus(); } -void SpdkStorageBackend::Shutdown() { +void SpdkBackend::Shutdown() { if (worker_ == nullptr) { return; } @@ -459,7 +459,7 @@ void SpdkStorageBackend::Shutdown() { worker_ = nullptr; } -absl::Status SpdkStorageBackend::RegisterFixedFiles(unsigned count) { +absl::Status SpdkBackend::RegisterFixedFiles(unsigned count) { if (worker_ == nullptr || count == 0 || !files_.empty()) { return absl::Status(absl::StatusCode::kFailedPrecondition, "invalid SPDK fixed-file table registration"); @@ -468,8 +468,7 @@ absl::Status SpdkStorageBackend::RegisterFixedFiles(unsigned count) { return absl::OkStatus(); } -absl::Status SpdkStorageBackend::RegisterBuffers( - std::span buffers) { +absl::Status SpdkBackend::RegisterBuffers(std::span buffers) { if (worker_ == nullptr || buffers.empty()) { return absl::Status(absl::StatusCode::kFailedPrecondition, "invalid SPDK DMA-buffer registration"); @@ -484,16 +483,15 @@ absl::Status SpdkStorageBackend::RegisterBuffers( return absl::OkStatus(); } -SpdkStorageBackend::OpenFile* SpdkStorageBackend::Lookup(FixedFile file) { +SpdkBackend::OpenFile* SpdkBackend::Lookup(FixedFile file) { return file.index_ < files_.size() && files_[file.index_].qpair_ != nullptr ? &files_[file.index_] : nullptr; } -absl::Status SpdkStorageBackend::SubmitOpenDirect(std::string_view path, - int flags, mode_t, - FixedFile file, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitOpenDirect(std::string_view path, int flags, + mode_t, FixedFile file, + IoCompletion* tag) { if ((flags & O_RDWR) == 0 || file.index_ >= files_.size() || tag == nullptr) { return absl::Status(absl::StatusCode::kInvalidArgument, "invalid SPDK open request"); @@ -534,8 +532,7 @@ absl::Status SpdkStorageBackend::SubmitOpenDirect(std::string_view path, return absl::OkStatus(); } -absl::Status SpdkStorageBackend::SubmitCloseDirect(FixedFile file, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitCloseDirect(FixedFile file, IoCompletion* tag) { OpenFile* opened = Lookup(file); if (opened == nullptr || tag == nullptr || outstanding_ != 0) { return absl::Status(absl::StatusCode::kFailedPrecondition, @@ -566,10 +563,9 @@ absl::Status SpdkStorageBackend::SubmitCloseDirect(FixedFile file, return absl::OkStatus(); } -absl::Status SpdkStorageBackend::SubmitIo(FixedFile file, void* buffer, - std::size_t bytes, - std::uint64_t offset, bool write, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitIo(FixedFile file, void* buffer, + std::size_t bytes, std::uint64_t offset, + bool write, IoCompletion* tag) { OpenFile* opened = Lookup(file); if (opened == nullptr || buffer == nullptr || bytes == 0 || tag == nullptr) { return absl::Status(absl::StatusCode::kInvalidArgument, @@ -597,37 +593,32 @@ absl::Status SpdkStorageBackend::SubmitIo(FixedFile file, void* buffer, return SubmitOrDefer(request, device->ns_, opened->channel_index_); } -absl::Status SpdkStorageBackend::SubmitReadFixed(FixedFile file, - FixedBuffer buffer, - std::uint64_t offset, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitReadFixed(FixedFile file, FixedBuffer buffer, + std::uint64_t offset, + IoCompletion* tag) { return SubmitIo(file, buffer.data_, buffer.size_, offset, false, tag); } -absl::Status SpdkStorageBackend::SubmitRead(FixedFile file, - std::span buffer, - std::uint64_t offset, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitRead(FixedFile file, + std::span buffer, + std::uint64_t offset, IoCompletion* tag) { return SubmitIo(file, buffer.data(), buffer.size(), offset, false, tag); } -absl::Status SpdkStorageBackend::SubmitWrite(FixedFile file, - std::span buffer, - std::uint64_t offset, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitWrite(FixedFile file, + std::span buffer, + std::uint64_t offset, IoCompletion* tag) { return SubmitIo(file, const_cast(buffer.data()), buffer.size(), offset, true, tag); } -absl::Status SpdkStorageBackend::SubmitWriteFixed(FixedFile file, - FixedBuffer buffer, - std::uint64_t offset, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitWriteFixed(FixedFile file, FixedBuffer buffer, + std::uint64_t offset, + IoCompletion* tag) { return SubmitIo(file, buffer.data_, buffer.size_, offset, true, tag); } -absl::Status SpdkStorageBackend::SubmitFdatasync(FixedFile file, - IoCompletion* tag) { +absl::Status SpdkBackend::SubmitFdatasync(FixedFile file, IoCompletion* tag) { OpenFile* opened = Lookup(file); if (opened == nullptr || tag == nullptr) { return absl::Status(absl::StatusCode::kInvalidArgument, @@ -651,8 +642,8 @@ absl::Status SpdkStorageBackend::SubmitFdatasync(FixedFile file, return SubmitOrDefer(request, device->ns_, opened->channel_index_); } -int SpdkStorageBackend::TrySubmit(const PendingIo& request, spdk_nvme_ns* ns, - spdk_nvme_qpair* qpair) { +int SpdkBackend::TrySubmit(const PendingIo& request, spdk_nvme_ns* ns, + spdk_nvme_qpair* qpair) { switch (request.operation_) { case PendingIo::Operation::kRead: return spdk_nvme_ns_cmd_read(ns, qpair, request.buffer_, request.lba_, @@ -668,9 +659,9 @@ int SpdkStorageBackend::TrySubmit(const PendingIo& request, spdk_nvme_ns* ns, return -EINVAL; } -absl::Status SpdkStorageBackend::SubmitOrDefer(const PendingIo& request, - spdk_nvme_ns* ns, - std::size_t channel_index) { +absl::Status SpdkBackend::SubmitOrDefer(const PendingIo& request, + spdk_nvme_ns* ns, + std::size_t channel_index) { const ControllerChannel& channel = channels_[channel_index]; if (!channel.pending_.empty()) [[unlikely]] { // Preserve submission order even if device capacity has become available @@ -689,7 +680,7 @@ absl::Status SpdkStorageBackend::SubmitOrDefer(const PendingIo& request, return SubmitError(rc); } -ABSL_ATTRIBUTE_NOINLINE absl::Status SpdkStorageBackend::QueuePending( +ABSL_ATTRIBUTE_NOINLINE absl::Status SpdkBackend::QueuePending( const PendingIo& request, std::size_t channel_index) { // SPDK retains nothing when submission fails. Store only unsubmitted // arguments here; issued requests use the caller's tag as callback context. @@ -705,7 +696,7 @@ ABSL_ATTRIBUTE_NOINLINE absl::Status SpdkStorageBackend::QueuePending( return absl::OkStatus(); } -ABSL_ATTRIBUTE_NOINLINE void SpdkStorageBackend::RetryDeferred( +ABSL_ATTRIBUTE_NOINLINE void SpdkBackend::RetryDeferred( std::size_t channel_index, unsigned limit, SpdkPollResult& result) { const std::size_t generation = channel_generation_; for (unsigned submitted = 0; !channels_[channel_index].pending_.empty() && @@ -737,7 +728,7 @@ ABSL_ATTRIBUTE_NOINLINE void SpdkStorageBackend::RetryDeferred( } } -SpdkPollResult SpdkStorageBackend::Poll(unsigned max_completions) { +SpdkPollResult SpdkBackend::Poll(unsigned max_completions) { assert(worker_ == nullptr || ThisWorker().self_ == worker_); SpdkPollResult result; if (!pending_completions_.empty()) { From 8cf9378e9acf4d633497bf2f95e78d089aa9bfd3 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 3 Nov 2026 04:52:04 +0000 Subject: [PATCH 3/3] refactor(io): share worker ring through CRTP facades --- docs/architecture/io.md | 10 +-- docs/architecture/runtime.md | 5 +- include/bycorf/io/net_backend.h | 104 ++++++++++++++------------- include/bycorf/io/storage_backend.h | 77 ++++++++++---------- include/bycorf/runtime/worker.h | 80 ++++++++++----------- src/io/spdk_storage.cpp | 4 +- src/runtime/worker.cpp | 56 +++++++-------- tests/io_uring_backpressure_test.cpp | 52 +++++++------- 8 files changed, 197 insertions(+), 191 deletions(-) diff --git a/docs/architecture/io.md b/docs/architecture/io.md index c11d97e..538b075 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -32,8 +32,10 @@ allocation or runtime startup fails. This preserves DMA allocator identity, registered-resource ownership and native TCP state. Unsupported capabilities fail explicitly rather than falling back. -`Worker` owns one `IoUringBackend`, borrowed by `NetBackend` and -`StorageBackend`. Kernel sockets, storage, timers and MSG_RING use the same +`Worker` owns one `IoUringBackend`, shared by its private CRTP facades +`NetBackend` and `StorageBackend`. The facades obtain the ring +through the owner contract rather than receiving it on each submission or +storing a separate pointer. Kernel sockets, storage, timers and MSG_RING use the same submission/completion queues. `NetBackend` selects kernel networking or its optional DPDK adapter; `StorageBackend` selects kernel storage or its embedded SPDK backend. SPDK default construction acquires no device resources; only the @@ -47,8 +49,8 @@ parking, while kernel storage uses the shared ring's wake/completion path. The worker initializes the ring before either adapter and stops storage, then networking, before shutting down the ring. Kernel storage registrations live -until ring shutdown quiesces kernel access. Both adapters are destroyed before -the borrowed ring; neither creates a second ring or worker pool. SPDK callbacks +until ring shutdown quiesces kernel access. The ring is the first base subobject, so both +facades are destroyed before it; neither creates a second ring or worker pool. SPDK callbacks resolve their backend through the storage adapter using the owning worker's TLS, including shutdown drains. diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 3df993d..20611d8 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -25,8 +25,9 @@ worker main function runs. An affinity or thread-launch failure prevents the worker main functions from entering partial service initialization. A `Worker` owns its coroutine queues, connections, shared kernel I/O backend, -network and storage adapters, and completion handling. Both adapters borrow the -kernel backend; storage-specific polling is encapsulated by `StorageBackend`. The scheduler bounds foreground and +network and storage facades, and completion handling. The facades are private +CRTP bases sharing the worker's kernel backend; storage-specific polling is +encapsulated by `StorageBackend`. The scheduler bounds foreground and background coroutine work and polls external completions between rounds. Workers submit queued foreground I/O and peer wakes before a maintenance slice, then submit any work queued by that slice before advancing to the next round. diff --git a/include/bycorf/io/net_backend.h b/include/bycorf/io/net_backend.h index aa7264c..5e505c2 100644 --- a/include/bycorf/io/net_backend.h +++ b/include/bycorf/io/net_backend.h @@ -25,23 +25,22 @@ #endif namespace bycorf { -// Routes networking through the worker-owned kernel ring or a DPDK adapter -// borrowing that same ring. Selection stays fixed after Init; the worker owns -// ring initialization and shutdown, including storage, timers and wakeups. +// CRTP network facade. Owner derives non-virtually from this specialization and +// grants it access to KernelBackend(), which returns the owner's shared ring. +// Downcasting computes a fixed address adjustment; no owner/ring pointer is +// stored here. Owner must keep the ring alive through adapter destruction. +template class NetBackend { public: - // The worker passes its ring directly so inlined kernel submissions do not - // load a stored ring pointer. DPDK borrows it for its adapter lifetime. - // Initialize networking after the worker initializes the shared ring. - absl::Status Init(IoUringBackend& kernel, const IoBackendOptions& options, - Worker* worker) { + // Initialize networking after the owner initializes the shared ring. DPDK + // borrows that ring for its adapter lifetime. + absl::Status Init(const IoBackendOptions& options, Worker* worker) { #if BYCORF_KERNEL_BYPASS if (DpdkNetworkEnabled()) { - dpdk_.emplace(kernel); + dpdk_.emplace(Kernel()); return dpdk_->Init(options, worker); } #endif - (void)kernel; (void)options; (void)worker; return absl::OkStatus(); @@ -63,78 +62,71 @@ class NetBackend { } #endif } - absl::Status SubmitSend(IoUringBackend& kernel, const RegisteredFile& file, + absl::Status SubmitSend(const RegisteredFile& file, std::span buffer, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitSend(file, buffer, tag); #endif - return kernel.SubmitSend(file, buffer, tag); + return Kernel().SubmitSend(file, buffer, tag); } - absl::Status SubmitSendMsg(IoUringBackend& kernel, const RegisteredFile& file, - const msghdr* message, IoCompletion* tag) { + absl::Status SubmitSendMsg(const RegisteredFile& file, const msghdr* message, + IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitSendMsg(file, message, tag); #endif - return kernel.SubmitSendMsg(file, message, tag); + return Kernel().SubmitSendMsg(file, message, tag); } - absl::Status SubmitAcceptMultishot(IoUringBackend& kernel, int listen_fd, - IoCompletion* tag) { + absl::Status SubmitAcceptMultishot(int listen_fd, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitAcceptMultishot(listen_fd, tag); #endif - return kernel.SubmitAcceptMultishot(listen_fd, tag); + return Kernel().SubmitAcceptMultishot(listen_fd, tag); } - absl::Status SubmitConnect(IoUringBackend& kernel, int fd, - const sockaddr* address, socklen_t address_length, - IoCompletion* tag) { + absl::Status SubmitConnect(int fd, const sockaddr* address, + socklen_t address_length, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitConnect(fd, address, address_length, tag); #endif - return kernel.SubmitConnect(fd, address, address_length, tag); + return Kernel().SubmitConnect(fd, address, address_length, tag); } - absl::Status SubmitCancel(IoUringBackend& kernel, IoCompletion* target) { + absl::Status SubmitCancel(IoCompletion* target) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitCancel(target); #endif - return kernel.SubmitCancel(target); + return Kernel().SubmitCancel(target); } // Cancel kernel socket operations that have not acquired their fd reference // yet. Native DPDK handles must be excluded by the caller. - void CancelUnsubmittedForFd(IoUringBackend& kernel, int fd) noexcept { - kernel.CancelUnsubmittedForFd(fd); + void CancelUnsubmittedForFd(int fd) noexcept { + Kernel().CancelUnsubmittedForFd(fd); } - absl::Status StartRecvMultishot(IoUringBackend& kernel, - Connection* connection) { + absl::Status StartRecvMultishot(Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->StartRecvMultishot(connection); #endif - return kernel.StartRecvMultishot(connection); + return Kernel().StartRecvMultishot(connection); } // Null tag requests cancellation without a separate completion consumer. - absl::Status SubmitCancelRecv(IoUringBackend& kernel, Connection* connection, - IoCompletion* tag) { + absl::Status SubmitCancelRecv(Connection* connection, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->SubmitCancelRecv(connection, tag); #endif - return kernel.SubmitCancelRecv(connection, tag); + return Kernel().SubmitCancelRecv(connection, tag); } - absl::Status StartPeerDisconnectPoll(IoUringBackend& kernel, - Connection* connection) { + absl::Status StartPeerDisconnectPoll(Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->StartPeerDisconnectPoll(connection); #endif - return kernel.StartPeerDisconnectPoll(connection); + return Kernel().StartPeerDisconnectPoll(connection); } - absl::Status CancelPeerDisconnectPoll(IoUringBackend& kernel, - Connection* connection) { + absl::Status CancelPeerDisconnectPoll(Connection* connection) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->CancelPeerDisconnectPoll(connection); #endif - return kernel.CancelPeerDisconnectPoll(connection); + return Kernel().CancelPeerDisconnectPoll(connection); } - std::span ViewRecvBuffer(const IoUringBackend& kernel, - const Connection* connection, + std::span ViewRecvBuffer(const Connection* connection, std::uint16_t buffer_id, std::size_t offset, std::size_t length) const { @@ -142,35 +134,47 @@ class NetBackend { if (dpdk_) return dpdk_->ViewRecvBuffer(connection, buffer_id, offset, length); #endif - return kernel.ViewRecvBuffer(connection, buffer_id, offset, length); + return Kernel().ViewRecvBuffer(connection, buffer_id, offset, length); } - void ReleaseRecvBuffer(IoUringBackend& kernel, Connection* connection, - std::uint16_t buffer_id) { + void ReleaseRecvBuffer(Connection* connection, std::uint16_t buffer_id) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->ReleaseRecvBuffer(connection, buffer_id); #endif - return kernel.ReleaseRecvBuffer(connection, buffer_id); + return Kernel().ReleaseRecvBuffer(connection, buffer_id); } - absl::Status Submit(IoUringBackend& kernel) { + absl::Status Submit() { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Submit(); #endif - return kernel.Submit(); + return Kernel().Submit(); } - bool Poll(IoUringBackend& kernel) { + bool Poll() { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Poll(); #endif - return kernel.Poll(); + return Kernel().Poll(); } - bool Wait(IoUringBackend& kernel, int timeout_ms) { + bool Wait(int timeout_ms) { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Wait(timeout_ms); #endif - return kernel.Wait(timeout_ms); + return Kernel().Wait(timeout_ms); } private: + // Only the declared owner may construct the facade; a standalone base + // would make the CRTP downcast invalid. Never access Owner in base teardown. + friend Owner; + NetBackend() = default; + ~NetBackend() = default; + + IoUringBackend& Kernel() noexcept { + return static_cast(*this).KernelBackend(); + } + const IoUringBackend& Kernel() const noexcept { + return static_cast(*this).KernelBackend(); + } + #if BYCORF_KERNEL_BYPASS // Inline the small adapter: packet/stream submissions already follow its // implementation pointer, so a separate heap adapter adds a dependent load diff --git a/include/bycorf/io/storage_backend.h b/include/bycorf/io/storage_backend.h index 161c00b..4c052f7 100644 --- a/include/bycorf/io/storage_backend.h +++ b/include/bycorf/io/storage_backend.h @@ -27,11 +27,12 @@ namespace bycorf { -// Routes worker-owned storage operations using the frozen process selection. -// Borrows the worker's kernel backend; neither initializes nor shuts down its -// ring. The worker passes the ring directly to submissions, avoiding a stored -// pointer load, and keeps it alive until both adapters have stopped. -// Accepted operations retain caller-owned tags/buffers until completion. +// CRTP storage facade. Owner derives non-virtually from this specialization and +// grants it access to KernelBackend(). The shared ring is reached by a fixed +// address adjustment, without a stored owner/ring pointer. Owner initializes +// and shuts down the ring, which must outlive the facade. Accepted operations +// retain caller-owned tags/buffers until completion. +template class StorageBackend { public: struct PollResult { @@ -39,7 +40,6 @@ class StorageBackend { std::uint32_t completions_ = 0; }; - StorageBackend() = default; StorageBackend(const StorageBackend&) = delete; StorageBackend& operator=(const StorageBackend&) = delete; @@ -87,93 +87,98 @@ class StorageBackend { return {}; } - absl::Status RegisterFixedFiles(IoUringBackend& kernel, unsigned count) { + absl::Status RegisterFixedFiles(unsigned count) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.RegisterFixedFiles(count); } #endif - return kernel.RegisterFixedFiles(count); + return Kernel().RegisterFixedFiles(count); } - absl::Status RegisterBuffers(IoUringBackend& kernel, - std::span buffers) { + absl::Status RegisterBuffers(std::span buffers) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.RegisterBuffers(buffers); } #endif - return kernel.RegisterBuffers(buffers); + return Kernel().RegisterBuffers(buffers); } - absl::Status SubmitOpenDirect(IoUringBackend& kernel, std::string_view path, - int flags, mode_t mode, FixedFile file, - IoCompletion* tag) { + absl::Status SubmitOpenDirect(std::string_view path, int flags, mode_t mode, + FixedFile file, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitOpenDirect(path, flags, mode, file, tag); } #endif - return kernel.SubmitOpenDirect(path, flags, mode, file, tag); + return Kernel().SubmitOpenDirect(path, flags, mode, file, tag); } - absl::Status SubmitCloseDirect(IoUringBackend& kernel, FixedFile file, - IoCompletion* tag) { + absl::Status SubmitCloseDirect(FixedFile file, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitCloseDirect(file, tag); } #endif - return kernel.SubmitCloseDirect(file, tag); + return Kernel().SubmitCloseDirect(file, tag); } - absl::Status SubmitReadFixed(IoUringBackend& kernel, FixedFile file, - FixedBuffer buffer, std::uint64_t offset, - IoCompletion* tag) { + absl::Status SubmitReadFixed(FixedFile file, FixedBuffer buffer, + std::uint64_t offset, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitReadFixed(file, buffer, offset, tag); } #endif - return kernel.SubmitReadFixed(file, buffer, offset, tag); + return Kernel().SubmitReadFixed(file, buffer, offset, tag); } - absl::Status SubmitRead(IoUringBackend& kernel, FixedFile file, - std::span buffer, std::uint64_t offset, - IoCompletion* tag) { + absl::Status SubmitRead(FixedFile file, std::span buffer, + std::uint64_t offset, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitRead(file, buffer, offset, tag); } #endif - return kernel.SubmitRead(file, buffer, offset, tag); + return Kernel().SubmitRead(file, buffer, offset, tag); } - absl::Status SubmitWrite(IoUringBackend& kernel, FixedFile file, - std::span buffer, + absl::Status SubmitWrite(FixedFile file, std::span buffer, std::uint64_t offset, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitWrite(file, buffer, offset, tag); } #endif - return kernel.SubmitWrite(file, buffer, offset, tag); + return Kernel().SubmitWrite(file, buffer, offset, tag); } - absl::Status SubmitWriteFixed(IoUringBackend& kernel, FixedFile file, - FixedBuffer buffer, std::uint64_t offset, - IoCompletion* tag) { + absl::Status SubmitWriteFixed(FixedFile file, FixedBuffer buffer, + std::uint64_t offset, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitWriteFixed(file, buffer, offset, tag); } #endif - return kernel.SubmitWriteFixed(file, buffer, offset, tag); + return Kernel().SubmitWriteFixed(file, buffer, offset, tag); } - absl::Status SubmitFdatasync(IoUringBackend& kernel, FixedFile file, - IoCompletion* tag) { + absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag) { #if BYCORF_KERNEL_BYPASS if (spdk_.IsInitialized()) { return spdk_.SubmitFdatasync(file, tag); } #endif - return kernel.SubmitFdatasync(file, tag); + return Kernel().SubmitFdatasync(file, tag); } private: + // Only the declared owner may construct the facade; a standalone base + // would make the CRTP downcast invalid. Never access Owner in base teardown. + friend Owner; + StorageBackend() = default; + ~StorageBackend() = default; + + IoUringBackend& Kernel() noexcept { + return static_cast(*this).KernelBackend(); + } + const IoUringBackend& Kernel() const noexcept { + return static_cast(*this).KernelBackend(); + } + #if BYCORF_KERNEL_BYPASS // Resolve callback ownership through worker TLS without adding an owner // pointer to every I/O request. diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index d274379..6b3b8c8 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -67,7 +67,15 @@ struct WorkerOptions { // The per-core scheduler: ready queue, connection table, cross-core mailbox and // the run loop. All io_uring mechanism lives in the backend it owns; the Worker // only forwards typed submissions to it and reacts to completions. -class Worker { +// Base order is also lifetime order: the shared ring is constructed first and +// destroyed last. Facade constructors/destructors must not downcast to Worker; +// Shutdown drains them while the complete worker and its queues are alive. +class Worker : private IoUringBackend, + private NetBackend, + private StorageBackend { + using Network = NetBackend; + using Storage = StorageBackend; + public: struct WakeStats { std::uint64_t checks_ = 0; @@ -229,95 +237,83 @@ class Worker { absl::Status SubmitSend(const RegisteredFile& file, std::span buffer, IoCompletion* tag) { - return networking_backend_.SubmitSend(kernel_backend_, file, buffer, tag); + return Network::SubmitSend(file, buffer, tag); } absl::Status SubmitSendMsg(const RegisteredFile& file, const msghdr* message, IoCompletion* tag) { - return networking_backend_.SubmitSendMsg(kernel_backend_, file, message, - tag); + return Network::SubmitSendMsg(file, message, tag); } absl::Status SubmitAcceptMultishot(int listen_fd, IoCompletion* tag) { - return networking_backend_.SubmitAcceptMultishot(kernel_backend_, listen_fd, - tag); + return Network::SubmitAcceptMultishot(listen_fd, tag); } // Async connect on a raw fd (pre-registration); see ConnectTcp. absl::Status SubmitConnect(int fd, const sockaddr* address, socklen_t address_length, IoCompletion* tag) { - return networking_backend_.SubmitConnect(kernel_backend_, fd, address, - address_length, tag); + return Network::SubmitConnect(fd, address, address_length, tag); } // Best-effort cancel of the in-flight SQE tagged `target`; owner-thread // only (the ring is single-issuer). absl::Status SubmitCancel(IoCompletion* target) { - return networking_backend_.SubmitCancel(kernel_backend_, target); + return Network::SubmitCancel(target); } // Before closing a kernel socket, retire accepted operations whose fd has // not yet been consumed by the kernel. Completions remain deferred. void CancelUnsubmittedForFd(int fd) noexcept { - networking_backend_.CancelUnsubmittedForFd(kernel_backend_, fd); + Network::CancelUnsubmittedForFd(fd); } absl::Status EnsureRecvArmed(Connection* connection) { - return networking_backend_.StartRecvMultishot(kernel_backend_, connection); + return Network::StartRecvMultishot(connection); } absl::Status SubmitCancelRecv(Connection* connection, IoCompletion* tag) { - return networking_backend_.SubmitCancelRecv(kernel_backend_, connection, - tag); + return Network::SubmitCancelRecv(connection, tag); } absl::Status EnsurePeerDisconnectPollArmed(Connection* connection) { - return networking_backend_.StartPeerDisconnectPoll(kernel_backend_, - connection); + return Network::StartPeerDisconnectPoll(connection); } absl::Status CancelPeerDisconnectPoll(Connection* connection) { - return networking_backend_.CancelPeerDisconnectPoll(kernel_backend_, - connection); + return Network::CancelPeerDisconnectPoll(connection); } std::span ViewMultishotBuffer(const Connection* connection, std::uint16_t buffer_id, std::size_t offset, std::size_t length) const { - return networking_backend_.ViewRecvBuffer(kernel_backend_, connection, - buffer_id, offset, length); + return Network::ViewRecvBuffer(connection, buffer_id, offset, length); } void ReleaseReceivedBuffer(Connection* connection, std::uint16_t buffer_id) { - networking_backend_.ReleaseRecvBuffer(kernel_backend_, connection, - buffer_id); + Network::ReleaseRecvBuffer(connection, buffer_id); } absl::Status RegisterFixedFiles(unsigned count) { - auto status = storage_backend_.RegisterFixedFiles(kernel_backend_, count); + auto status = Storage::RegisterFixedFiles(count); if (status.ok()) { storage_file_io_stats_.assign(count, StorageFileIoStats{}); } return status; } absl::Status RegisterBuffers(std::span buffers) { - return storage_backend_.RegisterBuffers(kernel_backend_, buffers); + return Storage::RegisterBuffers(buffers); } // Storage submissions retain the tag and referenced caller-owned buffers // until completion, including while queued. Each tag may have only one // unfinished storage operation; its completion may submit the next one. absl::Status SubmitOpenDirect(std::string_view path, int flags, mode_t mode, FixedFile file, IoCompletion* tag) { - return storage_backend_.SubmitOpenDirect(kernel_backend_, path, flags, mode, - file, tag); + return Storage::SubmitOpenDirect(path, flags, mode, file, tag); } absl::Status SubmitCloseDirect(FixedFile file, IoCompletion* tag) { - return storage_backend_.SubmitCloseDirect(kernel_backend_, file, tag); + return Storage::SubmitCloseDirect(file, tag); } absl::Status SubmitReadFixed(FixedFile file, FixedBuffer buffer, std::uint64_t offset, IoCompletion* tag) { - return storage_backend_.SubmitReadFixed(kernel_backend_, file, buffer, - offset, tag); + return Storage::SubmitReadFixed(file, buffer, offset, tag); } absl::Status SubmitRead(FixedFile file, std::span buffer, std::uint64_t offset, IoCompletion* tag) { - return storage_backend_.SubmitRead(kernel_backend_, file, buffer, offset, - tag); + return Storage::SubmitRead(file, buffer, offset, tag); } absl::Status SubmitWrite(FixedFile file, std::span buffer, std::uint64_t offset, IoCompletion* tag) { - auto status = storage_backend_.SubmitWrite(kernel_backend_, file, buffer, - offset, tag); + auto status = Storage::SubmitWrite(file, buffer, offset, tag); if (status.ok() && file.index_ < storage_file_io_stats_.size()) { storage_file_io_stats_[file.index_].submitted_write_bytes_ += buffer.size(); @@ -326,8 +322,7 @@ class Worker { } absl::Status SubmitWriteFixed(FixedFile file, FixedBuffer buffer, std::uint64_t offset, IoCompletion* tag) { - auto status = storage_backend_.SubmitWriteFixed(kernel_backend_, file, - buffer, offset, tag); + auto status = Storage::SubmitWriteFixed(file, buffer, offset, tag); if (status.ok() && file.index_ < storage_file_io_stats_.size()) { storage_file_io_stats_[file.index_].submitted_write_bytes_ += buffer.size_; @@ -335,11 +330,11 @@ class Worker { return status; } absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag) { - return storage_backend_.SubmitFdatasync(kernel_backend_, file, tag); + return Storage::SubmitFdatasync(file, tag); } absl::Status SubmitTimeout(const __kernel_timespec& timeout, IoCompletion* tag) { - return kernel_backend_.SubmitTimeout(timeout, tag); + return KernelBackend().SubmitTimeout(timeout, tag); } // Takes ownership of a connection with a valid socket; returns nullptr for @@ -368,6 +363,13 @@ class Worker { void SpawnBackground(Task task); private: + friend class NetBackend; + friend class StorageBackend; + // Shared-ring access for the CRTP facades; the base conversion is a fixed + // address adjustment rather than a load of a stored pointer. + IoUringBackend& KernelBackend() noexcept { return *this; } + const IoUringBackend& KernelBackend() const noexcept { return *this; } + friend class WorkerCrossCoreTestPeer; friend class WorkerSchedulerTestPeer; @@ -396,7 +398,7 @@ class Worker { bool ShouldRunBackground() const noexcept; void MaybeResetRuntimeWindow() noexcept; void RecordRound(std::int64_t start_cycles) noexcept; - void Flush(); // FlushWakes() + networking_backend_.Submit(kernel_backend_) + void Flush(); // FlushWakes() + Network::Submit() void FlushWakes(); // wake every marked, parked target once void CheckIdleConnections(); bool BusyPoll(); @@ -409,10 +411,6 @@ class Worker { bool DrainCrossCoreOverflow(); bool DrainCrossCore(); - // Declare the shared ring first so both borrowing adapters die before it. - IoUringBackend kernel_backend_{}; - NetBackend networking_backend_{}; - StorageBackend storage_backend_{}; bool initialized_ = false; // A fatal enter failure must leave RunOnce through the existing backend // failure path, rather than spinning forever on accepted pending work. diff --git a/src/io/spdk_storage.cpp b/src/io/spdk_storage.cpp index d300541..b1f888e 100644 --- a/src/io/spdk_storage.cpp +++ b/src/io/spdk_storage.cpp @@ -413,7 +413,7 @@ void SpdkBackend::CompleteAsync(void* context, spdk_nvme_cpl_is_error(completion) ? -EIO : tag->success_result_; Worker* worker = ThisWorker().self_; assert(worker != nullptr); - SpdkBackend* backend = &worker->storage_backend_.spdk_; + SpdkBackend* backend = &worker->StorageBackend::spdk_; // Complete may resubmit this tag or destroy it. Read its result first and // never access caller-owned state after dispatching the terminal completion. backend->CompleteRequest(tag, result); @@ -428,7 +428,7 @@ void SpdkBackend::CompleteRequest(IoCompletion* tag, int result) { } absl::Status SpdkBackend::Init(Worker* worker) { - if (worker == nullptr || &worker->storage_backend_.spdk_ != this) { + if (worker == nullptr || &worker->StorageBackend::spdk_ != this) { return absl::Status(absl::StatusCode::kInvalidArgument, "SPDK storage backend must be owned by its worker"); } diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index 2535d7c..5375a0b 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -209,26 +209,26 @@ absl::Status Worker::Init(const WorkerOptions& options) { auto kernel_options = backend_options; if (DpdkNetworkEnabled()) kernel_options.recv_buffer_count_ = 0; // Runtime pre-created the eventfd; the kernel backend borrows it. - auto status = kernel_backend_.Init(kernel_options, this, + auto status = KernelBackend().Init(kernel_options, this, cross_core_->mailbox(id_).wake_fd_); if (!status.ok()) { - networking_backend_.AbortStartup(status); + Network::AbortStartup(status); return status; } - status = networking_backend_.Init(kernel_backend_, backend_options, this); + status = Network::Init(backend_options, this); if (!status.ok()) { - kernel_backend_.Shutdown(); + KernelBackend().Shutdown(); return status; } - status = storage_backend_.Init(this); + status = Storage::Init(this); if (!status.ok()) { - networking_backend_.Shutdown(); - kernel_backend_.Shutdown(); + Network::Shutdown(); + KernelBackend().Shutdown(); return status; } // Advertise our ring so other workers can wake us via MSG_RING. - cross_core_->mailbox(id_).ring_fd_ = kernel_backend_.WakeHandle(); + cross_core_->mailbox(id_).ring_fd_ = KernelBackend().WakeHandle(); submit_failed_ = false; initialized_ = true; return absl::OkStatus(); @@ -238,9 +238,9 @@ void Worker::Shutdown() { if (!initialized_) { return; } - storage_backend_.Shutdown(); - networking_backend_.Shutdown(); - kernel_backend_.Shutdown(); + Storage::Shutdown(); + Network::Shutdown(); + KernelBackend().Shutdown(); initialized_ = false; } @@ -368,8 +368,7 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, // Stop the independent peer-disconnect observer before retiring the // Connection storage. Its completion owns one inflight operation and will // make the connection reclaimable. - (void)networking_backend_.CancelPeerDisconnectPoll(kernel_backend_, - connection); + (void)Network::CancelPeerDisconnectPoll(connection); // close(fd) alone does not release a socket held by an io_uring receive. // Cancel by operation identity before retiring it so the peer observes the @@ -377,8 +376,7 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, // Unlike shutdown(), this preserves a duplicated socket after PauseRead() // hands it to another worker. DPDK drains its corresponding receive here. if (connection->recv_armed_) { - (void)networking_backend_.SubmitCancelRecv(kernel_backend_, connection, - nullptr); + (void)Network::SubmitCancelRecv(connection, nullptr); } if (connection->file_.fd_ >= 0) { @@ -386,7 +384,7 @@ void Worker::BeginClose(Connection* connection, absl::Status reason, if (!detail::IsDpdkSocket(fd)) { // An accepted operation may still live in the software queue or SQ. // Retire it before close lets another socket reuse this descriptor. - networking_backend_.CancelUnsubmittedForFd(kernel_backend_, fd); + Network::CancelUnsubmittedForFd(fd); } connection->file_.fd_ = -1; connection->closed_ = true; @@ -804,8 +802,7 @@ void Worker::DiscardReceivedBuffers(Connection* connection) { // ReleaseRecvBuffer only recycles backend storage; it does not mutate this // queue or resume readers. Clear once to avoid shrinking while discarding. for (std::uint32_t i = 0; i < count; ++i) { - networking_backend_.ReleaseRecvBuffer(kernel_backend_, connection, - received[i].buffer_id_); + Network::ReleaseRecvBuffer(connection, received[i].buffer_id_); } received.clear(); } @@ -1054,7 +1051,7 @@ void Worker::FlushWakes() { #endif if (parked) { ++wake_sent_; - kernel_backend_.WakeRemote(mb.ring_fd_); + KernelBackend().WakeRemote(mb.ring_fd_); } } w.wake_list_.clear(); @@ -1068,7 +1065,7 @@ bool Worker::BusyPoll() { CycleNow() + static_cast(busy_poll_cycles_); do { bool did_work = DrainCrossCore(); - did_work |= networking_backend_.Poll(kernel_backend_); + did_work |= Network::Poll(); did_work |= PollStorage(); if (did_work) { const std::int64_t round_start = CycleNow(); @@ -1108,10 +1105,9 @@ bool Worker::PollStorage() { // nor deferred open/close completions pending, polling cannot make progress. // Use the same predicate as the idle/park decision so queued completions // still run even when the hardware outstanding count is zero. - if (!storage_backend_.NeedsPolling()) return false; + if (!Storage::NeedsPolling()) return false; const std::int64_t start = CycleNow(); - const auto result = - storage_backend_.Poll(options_.spdk_max_completions_per_poll_); + const auto result = Storage::Poll(options_.spdk_max_completions_per_poll_); const std::uint64_t elapsed = static_cast(CycleNow() - start); ++scheduler_stats_.storage_poll_calls_; scheduler_stats_.storage_poll_empty_ += result.completions_ == 0; @@ -1126,7 +1122,7 @@ bool Worker::PollStorage() { void Worker::Flush() { FlushWakes(); - const auto status = networking_backend_.Submit(kernel_backend_); + const auto status = Network::Submit(); if (!status.ok()) [[unlikely]] { // CQ backpressure and interrupted enters are retried by the next round. // A permanent error cannot make progress by waiting for queue capacity. @@ -1151,12 +1147,11 @@ bool Worker::RunOnce(bool wait_for_completion) { !foreground_remote_work_.empty() || !background_ready_.empty() || !next_background_ready_.empty() || !background_remote_work_.empty(); did_work |= DrainCrossCore(); - did_work |= networking_backend_.Poll(kernel_backend_); + did_work |= Network::Poll(); // This extra foreground slice exists to get ahead of storage completions. // Without pending storage work, the normal slice below already serves the // same queues; splitting it adds a second deadline and accounting pass. - if (storage_backend_.NeedsPolling() && - options_.spdk_foreground_pre_poll_us_ != 0 && + if (Storage::NeedsPolling() && options_.spdk_foreground_pre_poll_us_ != 0 && (!ready_.empty() || !next_ready_.empty() || !foreground_remote_work_.empty())) { MergeDeferred(); @@ -1231,7 +1226,7 @@ bool Worker::RunOnce(bool wait_for_completion) { if (submit_failed_) return false; // Submission can free SQ slots without producing a completion. Keep both // normal rounds and shutdown drains running until deferred work advances. - did_work |= kernel_backend_.HasPending(); + did_work |= KernelBackend().HasPending(); ReclaimConnections(); MaybeResetRuntimeWindow(); RecordRound(round_start); @@ -1248,7 +1243,7 @@ bool Worker::RunOnce(bool wait_for_completion) { } // A polled storage backend has no kernel wake source for its completions. - if (storage_backend_.NeedsPolling()) { + if (Storage::NeedsPolling()) { return true; } @@ -1273,8 +1268,7 @@ bool Worker::RunOnce(bool wait_for_completion) { } const int timeout_ms = options_.idle_timeout_ms_ > 0 ? 100 : -1; - const bool ok = networking_backend_.Wait( - kernel_backend_, timeout_ms); // blocks; dispatches on wake + const bool ok = Network::Wait(timeout_ms); // blocks; dispatches on wake mb.wake_seq_.store(0, std::memory_order_release); // leave the parked state return ok; // next iteration drains what Wait dispatched } diff --git a/tests/io_uring_backpressure_test.cpp b/tests/io_uring_backpressure_test.cpp index 46413a6..790851c 100644 --- a/tests/io_uring_backpressure_test.cpp +++ b/tests/io_uring_backpressure_test.cpp @@ -404,27 +404,31 @@ TEST(IoUringBackpressure, StorageReads) { // delivers storage completions, and stopping either facade must leave the // worker-owned ring available until the worker shuts it down. TEST(IoUringBackpressure, NetworkAndStorageShareRing) { - Fixture fixture; - ASSERT_NO_FATAL_FAILURE(fixture.Init()); - struct NetworkLoop { - bycorf::NetBackend network; - bycorf::IoUringBackend& kernel; - absl::Status Submit() { return network.Submit(kernel); } - bool Poll() { return network.Poll(kernel); } - } loop{{}, fixture.backend}; - auto& network = loop.network; - bycorf::StorageBackend storage; - ASSERT_TRUE(network.Init(fixture.backend, {}, &fixture.worker).ok()); - ASSERT_TRUE(storage.Init(&fixture.worker).ok()); + struct BackendOwner : bycorf::IoUringBackend, + bycorf::NetBackend, + bycorf::StorageBackend { + bycorf::IoUringBackend& KernelBackend() noexcept { return *this; } + const bycorf::IoUringBackend& KernelBackend() const noexcept { + return *this; + } + }; + bycorf::Worker worker; + BackendOwner owner; + auto& kernel = owner.KernelBackend(); + auto& network = static_cast&>(owner); + auto& storage = static_cast&>(owner); + ASSERT_TRUE( + kernel.Init({.ring_entries_ = 2, .recv_buffer_count_ = 0}, &worker, -1) + .ok()); + ASSERT_TRUE(network.Init({}, &worker).ok()); + ASSERT_TRUE(storage.Init(&worker).ok()); ASSERT_FALSE(storage.NeedsPolling()); - ASSERT_TRUE(storage.RegisterFixedFiles(fixture.backend, 1).ok()); + ASSERT_TRUE(storage.RegisterFixedFiles(1).ok()); Completion opened; - ASSERT_TRUE(storage - .SubmitOpenDirect(fixture.backend, "/dev/zero", O_RDONLY, 0, - {0}, &opened) - .ok()); - Drain(loop, [&] { return opened.calls != 0; }); + ASSERT_TRUE( + storage.SubmitOpenDirect("/dev/zero", O_RDONLY, 0, {0}, &opened).ok()); + Drain(network, [&] { return opened.calls != 0; }); ASSERT_EQ(opened.result, 0); int sockets[2]; @@ -435,15 +439,13 @@ TEST(IoUringBackpressure, NetworkAndStorageShareRing) { bytes.fill(std::byte{1}); const std::array payload{std::byte{42}}; Completion read, sent; - ASSERT_TRUE(storage.SubmitRead(fixture.backend, {0}, bytes, 0, &read).ok()); - ASSERT_TRUE( - network.SubmitSend(fixture.backend, {.fd_ = sockets[0]}, payload, &sent) - .ok()); + ASSERT_TRUE(storage.SubmitRead({0}, bytes, 0, &read).ok()); + ASSERT_TRUE(network.SubmitSend({.fd_ = sockets[0]}, payload, &sent).ok()); ASSERT_EQ(read.calls, 0u); ASSERT_EQ(sent.calls, 0u); // Kernel storage never requests a separate poll or keeps the worker spinning. ASSERT_FALSE(storage.NeedsPolling()); - Drain(loop, [&] { return read.calls != 0 && sent.calls != 0; }); + Drain(network, [&] { return read.calls != 0 && sent.calls != 0; }); ASSERT_EQ(read.result, bytes.size()); ASSERT_EQ(sent.result, 1); EXPECT_EQ(bytes, (std::array{})); @@ -456,8 +458,8 @@ TEST(IoUringBackpressure, NetworkAndStorageShareRing) { storage.Shutdown(); network.Shutdown(); Completion closed; - ASSERT_TRUE(fixture.backend.SubmitCloseDirect({0}, &closed).ok()); - Drain(fixture.backend, [&] { return closed.calls != 0; }); + ASSERT_TRUE(kernel.SubmitCloseDirect({0}, &closed).ok()); + Drain(kernel, [&] { return closed.calls != 0; }); ASSERT_EQ(closed.result, 0); }