diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bd415a..bad842d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,8 @@ if (BYCORF_BUILD_TESTS) tests/connection_storage_test.cpp tests/http_service_test.cpp tests/tcp_buffered_read_test.cpp - tests/io_uring_backpressure_test.cpp) + tests/io_uring_backpressure_test.cpp + tests/worker_wait_test.cpp) target_link_libraries(bycorf_runtime_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_runtime_tests 90 "runtime;net;io_uring") @@ -525,7 +526,7 @@ if (BYCORF_BUILD_TESTS) ${BYCORF_DPDK_COMPILE_FLAGS}) foreach(symbol IN ITEMS rte_eth_dev_configure rte_eth_dev_info_get rte_eth_dev_rx_intr_ctl_q_get_fd rte_eth_dev_rx_intr_enable - rte_eth_dev_rx_intr_disable) + rte_eth_dev_rx_intr_disable bycorf_bsd_deadline_ns) target_link_options(bycorf_dpdk_adaptive_tests PRIVATE "LINKER:--wrap=${symbol}") endforeach() diff --git a/README.md b/README.md index 44a19dc..ed89e60 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ results. |---|---| | `InlineRingQueue`, `ReceivedBufferQueue` | Queue types and inline capacities, moves, FIFO order, wrapping, growth, and shrinking | | `CrossCoreDrain`, `CrossCoreDrainState` | Shared sender quotas across message kinds, overflow fairness, bitmap wrapping, and concurrent publication without lost work | +| `WorkerWait` | Publication between idle rechecks and blocking, MSG_RING/eventfd wakes, stop races, and runnable work preventing sleep | | `WorkerScheduler` | Foreground/background FIFO admission, deferred turns, and reentrant local/remote work across queue relocation | | `TaskTransfer` | Bounded native stack for sequential and nested awaits, asynchronous completion, cancellation ownership and reentrant callbacks | | `RuntimeStartup`, `TcpServiceStop`, `Timer`, `Connect` | Early shutdown, timers and cancellation, loopback TCP, refused connections, deadlines, address validation, and retirement of losing deadlines | diff --git a/docs/architecture/README.md b/docs/architecture/README.md index a5ffa4c..ec2f8bb 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -22,7 +22,7 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging. | Module | Responsibility | Document | |---|---|---| -| Runtime | Worker lifecycle, coroutine scheduling, cross-worker and foreign ingress | [Runtime](runtime.md) | +| Runtime | Worker lifecycle, coroutine scheduling, sleep coordination, cross-worker and foreign ingress | [Runtime](runtime.md) | | I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | | Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) | | RPC | Framed request/reply protocol above streams | [RPC](rpc.md) | diff --git a/docs/architecture/io.md b/docs/architecture/io.md index 538b075..9ad7fb5 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -16,11 +16,11 @@ limitations under the License. # I/O backends -Every worker retains one io_uring instance for ordinary file I/O, kernel -sockets, completion delivery, timers, and cross-worker MSG_RING wakeups. -Registered storage buffers and network receive buffers have separate -lifecycles. The default network backend uses Linux TCP and provided-buffer -multishot receives, with a per-connection one-shot fallback. +`Worker` owns one `IoUringBackend` for kernel sockets, file I/O, completion +delivery, timers, and MSG_RING/eventfd wakeups. Its `NetBackend` and +`StorageBackend` facades share that ring. Network receive buffers and +registered storage buffers have separate lifecycles. Linux TCP uses +provided-buffer multishot receives with a per-connection one-shot fallback. `BYCORF_KERNEL_BYPASS` is the single build option for DPDK networking and SPDK storage. It defaults to off; enabling it compiles and links both capabilities. @@ -32,27 +32,21 @@ allocation or runtime startup fails. This preserves DMA allocator identity, registered-resource ownership and native TCP state. Unsupported capabilities fail explicitly rather than falling back. -`Worker` owns one `IoUringBackend`, shared by its private CRTP facades -`NetBackend` and `StorageBackend`. The facades obtain the ring -through the owner contract rather than receiving it on each submission or -storing a separate pointer. Kernel sockets, storage, timers and MSG_RING use the same -submission/completion queues. `NetBackend` selects kernel networking or its -optional DPDK adapter; `StorageBackend` selects kernel storage or its embedded -SPDK backend. SPDK default construction acquires no device resources; only the -selected backend is initialized. Storage dispatch uses its worker-local -initialized state, which remains active through shutdown completion draining. -Storage operations require a successfully initialized worker and stop before -its backend shutdown completes. The worker forwards storage operations without -selecting SPDK itself. Network event-loop polling and submission advance the -shared ring; separate storage polling advances only SPDK. Pending polled storage prevents -parking, while kernel storage uses the shared ring's wake/completion path. - -The worker initializes the ring before either adapter and stops storage, then -networking, before shutting down the ring. Kernel storage registrations live -until ring shutdown quiesces kernel access. The ring is the first base subobject, so both -facades are destroyed before it; neither creates a second ring or worker pool. SPDK callbacks -resolve their backend through the storage adapter using the owning worker's TLS, -including shutdown drains. +`NetBackend` selects kernel networking or the DPDK adapter; `StorageBackend` +selects kernel storage or SPDK. Only selected adapters acquire resources. +The worker explicitly polls the shared ring, native network work, and polled +storage. It flushes native network output before submitting the shared ring. +Network polling and flushing advance only DPDK work; storage polling advances +only SPDK. Kernel TCP and storage progress through the worker's ring calls. +Network adapters supply notification preparation, idle checks, +deadlines, and cleanup to the [worker sleep protocol](runtime.md#sleep-and-wakeups). +The worker enters the ring's blocking wait; adapters never block independently. + +The ring outlives both adapters. The worker initializes it first, stops storage +and networking in that order, then shuts it down. Storage dispatch remains +valid through shutdown completion draining; SPDK callbacks use the owning +worker's TLS to reach its storage adapter. Kernel registrations remain live +until ring shutdown quiesces kernel access. Each io_uring backend owns a FIFO for accepted operations awaiting submission space. Successful admission does not mean the kernel has received the request. @@ -60,8 +54,8 @@ The backend retains its prepared descriptor; the caller retains the completion tag, referenced buffers and parameter storage, and any registered resources until the operation's terminal completion. Submission and completion polling advance the queue on its owner worker, without dispatching completions inside -an operation's submission call. The event loop advances queued work before -parking, including when no earlier request can produce a completion. +an operation's submission call. Unsubmitted operations veto worker sleep even +when no earlier request can produce a completion. Cancellation preserves this ownership boundary: a queued operation retires through deferred completion, while an issued operation remains live until its @@ -82,14 +76,14 @@ unfinished storage operation has a distinct tag, which retains its expected successful result; the caller keeps the tag and buffers alive until completion. The completion callback may reuse or destroy the tag. Polling and shutdown drains retain the owning worker's thread-local context for completion dispatch. -Deferred commands participate in outstanding-I/O accounting, preventing -premature sleep, close or shutdown. Submission errors that cannot make progress -through a completion are reported to the caller. -The io_uring selection uses ordinary aligned buffers and kernel file/block I/O. -SPDK completion polling is inactive when storage uses io_uring; in-flight SPDK -storage prevents sleeping before its completions can be polled. As with the -NVMe FLUSH command, SPDK durability barriers cover previously completed writes; -callers await data writes before requesting a barrier. Controllers that report +Accepted and deferred SPDK commands require polling and veto worker sleep +until they retire; they also participate in close and shutdown accounting. +Submission errors that cannot make progress through a completion are reported +to the caller. Kernel storage uses ordinary aligned buffers and the shared +ring's completion path without SPDK polling. + +As with the NVMe FLUSH command, SPDK durability barriers cover previously +completed writes; callers await data writes before requesting a barrier. Controllers that report no volatile write cache already make completed writes durable, so their no-op FLUSH completes through the worker's deferred completion queue. diff --git a/docs/architecture/networking.md b/docs/architecture/networking.md index 5f07b5b..6f60273 100644 --- a/docs/architecture/networking.md +++ b/docs/architecture/networking.md @@ -157,15 +157,16 @@ are bounded: overflow and partial device TX free unaccepted packets, leaving native TCP responsible for retransmission. This is a functional integration boundary, not a zero-copy or tuned packet allocator. -## Polling and sleep +## DPDK notifications -Poll mode keeps workers running without arming RX interrupts. Adaptive mode -can wait on the same io_uring used for worker messages. A queue owner enables -its RX interrupt, registers the notification descriptor with io_uring, and -rechecks packet and software queues before sleeping. Packet publication either -prevents sleep or sends a MSG_RING wake to the parked destination. The next -BSD timer deadline bounds the wait; an arriving packet does not need to wait -for a timer tick. RX interrupts are disabled on returning to polling. +DPDK participates in the [worker sleep protocol](runtime.md#sleep-and-wakeups). +Explicit poll mode vetoes sleep without arming interrupts. Adaptive preparation +marks the software lane parked and, for a queue owner, enables RX interrupts +and registers the notification descriptor with the shared io_uring. Packet +publication either appears in the RX/TX queue recheck or sends MSG_RING to the +parked destination. The FreeBSD timer deadline bounds the worker's wait. +Cleanup clears the lane's parked state and disables any interrupt enabled by +that attempt, including on early exit or partial preparation failure. A TAP virtual device provides a readable packet descriptor instead of a physical interrupt eventfd. The notification code preserves that distinction: @@ -175,12 +176,11 @@ after a timer or mailbox wake does not indicate lost interrupt support. If the PMD cannot provide the required notification capability on every RX queue, adaptive startup fails before services start. Configuration, notification registration, and interrupt-arm failures never silently select polling; the -error identifies an explicit `BYCORF_DPDK_MODE=poll` override. A notification -failure during a wait terminates the worker loop and stops peer workers before -teardown barriers, producing a nonzero server exit. Virtual-device tests cover -this wake protocol but do not validate physical NIC interrupt behavior or predict kernel-bypass throughput. The TAP -PMD additionally uses Linux realtime signals for its internal RX trigger in -both modes; this is separate from Bycorf's worker sleep protocol. +error identifies an explicit `BYCORF_DPDK_MODE=poll` override. Notification +completion and interrupt-cleanup failures use the runtime's worker-failure +shutdown path. The TAP PMD uses Linux realtime signals for its internal RX +trigger in both modes. Virtual-device tests cover the notification protocol, but do not +validate physical NIC interrupt behavior or predict kernel-bypass throughput. Sources: `include/bycorf/net/connection.h`, `src/net/tcp_service.cpp`, `src/net/tcp_listener.cpp`, `src/net/tcp_stream.cpp`, `src/net/socket_ops.cpp`, `src/io/dpdk_backend.cpp`, diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index 20611d8..ad6ac73 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -24,16 +24,16 @@ inherited mask. Every thread completes affinity setup before any worker main function runs. An affinity or thread-launch failure prevents the worker main functions from entering partial service initialization. -A `Worker` owns its coroutine queues, connections, shared kernel I/O backend, -network and storage facades, and completion handling. The facades are private -CRTP bases sharing the worker's kernel backend; storage-specific polling is -encapsulated by `StorageBackend`. The scheduler bounds foreground and -background coroutine work and polls external completions between rounds. -Workers submit queued foreground I/O and peer wakes before a maintenance slice, -then submit any work queued by that slice before advancing to the next round. -Coroutines cooperate: a long non-suspending handler can delay all work on its -worker. Worker-local synchronization and cross-worker synchronization have -separate ownership contracts. +A `Worker` owns its coroutine queues, connections, completion handling, and +[shared I/O backend and adapters](io.md). Each round polls completions and +runs bounded foreground and background coroutine work. The worker polls the +shared ring before native network work and polled storage. Before maintenance +and at the end of the round, it queues peer wakes, flushes native network +output, and submits the shared ring. Idle rechecks use the same ordering; +facades do not poll or submit the ring on the worker's behalf. Coroutines +cooperate: a non-suspending handler can +delay all work on its worker. Worker-local and cross-worker synchronization +have separate ownership contracts. A worker resumes a ready coroutine directly. `Task` enters its child and returns to its parent continuation by symmetric transfer, within the same @@ -46,12 +46,32 @@ the selected toolchain and build configuration. Awaited child frames remain owned by the awaiting `Task` object, including when shutdown destroys a suspended root and its child chain. -Cross-worker submissions enter destination mailboxes and resume waiting -coroutines on their original workers. Workers use io_uring MSG_RING to wake a -peer; foreign ingress and process control use the pre-created eventfds. -`ForeignExecutor` is non-owning and is valid only while its runtime accepts -work. Stop closes ingress and wakes workers through descriptors that remain -alive until after their threads have joined. +## Sleep and wakeups + +The worker alone decides when its thread can block. It prepares source +notifications, rechecks I/O, mailbox and scheduler work, completes the mailbox +wake-sequence handshake, and submits operations and peer wakes. After a final +idle check it waits on the shared io_uring. Runnable work, a stop request, +unsubmitted operations, or storage requiring polling prevent sleep. Network +deadlines and enabled connection-idle checks bound the wait. + +Cross-worker submissions enter destination mailboxes and resume coroutines on +their original workers. The mailbox handshake makes racing publication either +prevent sleep or leave a persistent wake: workers use MSG_RING, while foreign +ingress and process control use pre-created eventfds. Consuming a completion +can make work runnable or observe a stop request, so rechecking must account +for both before blocking. + +Every wait attempt clears its mailbox parked state and ends source notification +preparation, including attempts abandoned before blocking. Source cleanup +releases per-attempt state; outstanding notification polls retain their tags +and descriptors through completion or ring shutdown. + +`ForeignExecutor` is non-owning and valid while its runtime accepts work. Stop +closes ingress and wakes workers through descriptors that remain alive until +after their threads have joined. + +## Shutdown and integration `Server` starts each registered service on its selected global worker IDs; an empty selection means every worker. Prepare sees the full runtime count, @@ -60,23 +80,25 @@ workers first leave their loops, then quiesce I/O and reclaim coroutine frames on their owning native threads. Service finalizers run after all worker frames are reclaimed. A worker that fails initialization contributes to both teardown barriers so initialized peers can finish. Temporary I/O submission shortages -are retried by the worker loop. A permanent submission error or unexpected -backend wait failure stops peer workers before entering those barriers and -yields a nonzero exit. Services outlive the server. +are retried by the worker loop. A permanent submission error, notification +preparation or cleanup failure, or shared-ring wait failure stops peer workers +before entering those barriers and yields a nonzero exit. Services outlive +the server. With DPDK networking, runtime preparation configures the shared port before launch. Worker 0 initializes the native FreeBSD kernel services; remaining -workers attach separate VNETs in serialized startup. The existing worker loop +workers attach separate VNETs in serialized startup. The worker loop runs packet input, TCP timers, and deferred BSD tasks. EAL may have control or interrupt helper threads; it does not launch Bycorf network polling workers. Worker-affine BSD sockets close and EAL thread registrations detach before join; the runtime then releases the port and software packet queues. -Sources: `src/runtime/runtime.cpp`, `src/runtime/worker.cpp`, -`include/bycorf/runtime/task.h`, `include/bycorf/runtime/cross_core.h`, `src/runtime/foreign_executor.cpp`, -`src/net/server.cpp`. - Active registered streams are counted across all workers and runtimes in the process. The atomic snapshot includes outgoing control connections and is updated only at connection open/close, so monitoring can read it without scheduling work on a control worker. + +Sources: `src/runtime/runtime.cpp`, `src/runtime/worker.cpp`, +`src/runtime/worker_wait.h`, `include/bycorf/runtime/task.h`, +`include/bycorf/runtime/cross_core.h`, `src/runtime/foreign_executor.cpp`, +`src/net/server.cpp`. diff --git a/include/bycorf/io/dpdk_backend.h b/include/bycorf/io/dpdk_backend.h index 70a39c3..2219997 100644 --- a/include/bycorf/io/dpdk_backend.h +++ b/include/bycorf/io/dpdk_backend.h @@ -19,6 +19,7 @@ #include +#include "absl/status/statusor.h" #include "bycorf/io/io_uring_backend.h" namespace bycorf { @@ -53,9 +54,22 @@ class DpdkBackend { // The worker initializes the borrowed ring before attaching networking. absl::Status Init(const IoBackendOptions&, Worker*); void Shutdown(); + // Drive native packet/TCP work only; the worker separately polls io_uring. bool Poll(); - bool Wait(int timeout_ms); - absl::Status Submit(); + // Owner-thread sleep protocol. Every PrepareWait call, including a partial + // failure or a polling veto, must be paired with FinishWait before retrying. + // True allows the worker to recheck all sources; false requires polling. + absl::StatusOr PrepareWait(); + // Call after Poll: checks residual software work and notification health. + bool CanWait() const; + // Bound the caller's timeout by the current BSD deadline (-1 is infinite). + int WaitTimeout(int timeout_ms) const; + // Release this attempt's parked state and RX interrupt, even if no wait ran. + // Outstanding notification polls retain their tag/fd until completion. + // waited records entry to the shared ring wait for diagnostic accounting. + absl::Status FinishWait(bool waited); + // Flush native TX work without submitting the borrowed kernel ring. + void Flush(); absl::Status SubmitSend(const RegisteredFile&, std::span, IoCompletion*); absl::Status SubmitConnect(int fd, const sockaddr*, socklen_t, IoCompletion*); diff --git a/include/bycorf/io/io_uring_backend.h b/include/bycorf/io/io_uring_backend.h index 8677d25..7fe46f0 100644 --- a/include/bycorf/io/io_uring_backend.h +++ b/include/bycorf/io/io_uring_backend.h @@ -141,8 +141,12 @@ class IoUringBackend { // Event loop. absl::Status Submit(); - bool Poll(); // reap completions, dispatch, then re-arm recvs - bool Wait(int timeout_ms); // block for a completion, dispatch, drain + bool Poll(); // reap completions, dispatch, then re-arm recvs + // The worker coordinates source notification arming and idle rechecks before + // calling this sole blocking primitive. Pending submissions veto blocking; + // an existing CQE returns immediately. Dispatch can enqueue runnable work + // or consume a stop notification, so the caller must re-enter scheduling. + bool Wait(int timeout_ms); // Software work or SQEs still requiring an enter must prevent worker sleep. bool HasPending() const noexcept; diff --git a/include/bycorf/io/net_backend.h b/include/bycorf/io/net_backend.h index 5e505c2..138e121 100644 --- a/include/bycorf/io/net_backend.h +++ b/include/bycorf/io/net_backend.h @@ -142,23 +142,46 @@ class NetBackend { #endif return Kernel().ReleaseRecvBuffer(connection, buffer_id); } - absl::Status Submit() { + // Advance only native network work. The owner drives the shared kernel + // ring separately, including when kernel TCP is selected. + void Flush() { #if BYCORF_KERNEL_BYPASS - if (dpdk_) return dpdk_->Submit(); + if (dpdk_) dpdk_->Flush(); #endif - return Kernel().Submit(); } bool Poll() { #if BYCORF_KERNEL_BYPASS if (dpdk_) return dpdk_->Poll(); #endif - return Kernel().Poll(); + return false; } - bool Wait(int timeout_ms) { + // Owner-thread preparation is always paired with FinishWait, including + // errors/vetoes. Kernel sockets already have persistent completion sources. + absl::StatusOr PrepareWait() { #if BYCORF_KERNEL_BYPASS - if (dpdk_) return dpdk_->Wait(timeout_ms); + if (dpdk_) return dpdk_->PrepareWait(); #endif - return Kernel().Wait(timeout_ms); + return true; + } + // Recheck after Poll and submission; this does not consume completions. + bool CanWait() const { +#if BYCORF_KERNEL_BYPASS + if (dpdk_) return dpdk_->CanWait(); +#endif + return true; + } + int WaitTimeout(int timeout_ms) const { +#if BYCORF_KERNEL_BYPASS + if (dpdk_) return dpdk_->WaitTimeout(timeout_ms); +#endif + return timeout_ms; + } + absl::Status FinishWait(bool waited) { +#if BYCORF_KERNEL_BYPASS + if (dpdk_) return dpdk_->FinishWait(waited); +#endif + (void)waited; + return absl::OkStatus(); } private: diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index a3168e3..2be3f80 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -375,6 +375,7 @@ class Worker : private IoUringBackend, friend class WorkerCrossCoreTestPeer; friend class WorkerSchedulerTestPeer; + friend class WorkerWaitTestPeer; // All message kinds share one sender visit and a combined consumption quota. // Retain its position across polls until it empties or exhausts that quota. @@ -401,10 +402,16 @@ class Worker : private IoUringBackend, bool ShouldRunBackground() const noexcept; void MaybeResetRuntimeWindow() noexcept; void RecordRound(std::int64_t start_cycles) noexcept; - void Flush(); // FlushWakes() + Network::Submit() + void Flush(); // Peer wakes + native network output + shared ring submit void FlushWakes(); // wake every marked, parked target once void CheckIdleConnections(); bool BusyPoll(); + bool HasRunnableWork() const noexcept; + // Observer is a compile-time no-op in production. Tests instantiate this + // same protocol with deterministic publication at its race boundaries. + enum class WaitPhase { kPrepared, kRechecked, kParked, kSubmitted, kWaiting }; + template + bool WaitForWork(Observer&& observe); bool PollStorage(); bool CanReclaim(const Connection& connection) const noexcept; void ReclaimConnections(); diff --git a/src/io/dpdk_backend.cpp b/src/io/dpdk_backend.cpp index e2ada84..736fee9 100644 --- a/src/io/dpdk_backend.cpp +++ b/src/io/dpdk_backend.cpp @@ -250,6 +250,7 @@ class DpdkBackend::Impl { std::uint64_t rx = 0, tx = 0, forwarded_rx = 0, forwarded_tx = 0, drops = 0; std::uint64_t waits = 0, arms = 0, notifications = 0; bool rx_can_wait = false; + bool wait_prepared = false, rx_interrupt_enabled = false; struct Notification : IoCompletion { Impl* owner = nullptr; @@ -299,9 +300,9 @@ class DpdkBackend::Impl { } void Wake(unsigned target) { Lane& lane = fabric.lanes[target]; - // Publication precedes this exchange. Wait marks parked before rechecking - // its rings, so work either prevents the wait or has a MSG_RING wake - // queued. + // Publication precedes this exchange. PrepareWait marks parked before + // rechecking its rings, so work either prevents the wait or has a MSG_RING + // wake queued. if (target != id && lane.parked.exchange(false, std::memory_order_acq_rel)) backend.kernel_.WakeRemote(lane.ring_fd); } @@ -950,93 +951,83 @@ void DpdkBackend::Shutdown() { bool DpdkBackend::Poll() { if (!impl_->initialized || impl_->polling) return false; impl_->polling = true; - const bool kernel = kernel_.Poll(); const bool packets = impl_->Packets(); const bool sockets = impl_->SocketCompletions(); impl_->polling = false; - return kernel || packets || sockets; + return packets || sockets; } -absl::Status DpdkBackend::Submit() { +void DpdkBackend::Flush() { if (impl_->initialized) impl_->FlushTx(); - return kernel_.Submit(); } -bool DpdkBackend::Wait(int timeout_ms) { +absl::StatusOr DpdkBackend::PrepareWait() { if (!fabric.adaptive || fabric.ready.load(std::memory_order_acquire) != fabric.workers) - return true; - if (!impl_->rx_can_wait) return false; + return false; + if (!impl_->rx_can_wait) + return absl::InternalError("DPDK RX notification is unavailable"); const unsigned id = impl_->id; - Lane& lane = fabric.lanes[id]; - lane.parked.store(true, std::memory_order_release); - bool enabled = false; + // RMW pairs with the producer's post-publication exchange. If the producer + // ran first, acquire observes its queue publication; otherwise it observes + // parked and sends MSG_RING. A release store alone does not order the + // following queue loads against the producer on every supported CPU. + fabric.lanes[id].parked.exchange(true, std::memory_order_acq_rel); + impl_->wait_prepared = true; if (id < fabric.queues) { if (!fabric.tap) { const int rc = rte_eth_dev_rx_intr_enable(fabric.port, id); - enabled = rc == 0; - if (!enabled) { - lane.parked.store(false); - std::fprintf( - stderr, "DPDK worker %u: %s\n", id, - AdaptiveError("enable RX interrupt", rc).ToString().c_str()); - return false; - } + if (rc) return AdaptiveError("enable RX interrupt", rc); + impl_->rx_interrupt_enabled = true; } ++impl_->arms; if (!impl_->notification.armed) { - const auto submitted = kernel_.SubmitPoll(impl_->notification.fd, POLLIN, - &impl_->notification); - if (!submitted.ok()) { - lane.parked.store(false); - if (enabled) DisableRxInterrupt(id); - std::fprintf(stderr, - "DPDK worker %u: RX notification registration: %s; " - "set BYCORF_DPDK_MODE=poll explicitly to use polling\n", - id, submitted.ToString().c_str()); - return false; - } + const auto status = kernel_.SubmitPoll(impl_->notification.fd, POLLIN, + &impl_->notification); + if (!status.ok()) + return absl::Status( + status.code(), + "DPDK RX notification registration: " + + std::string(status.message()) + + "; set BYCORF_DPDK_MODE=poll explicitly to use polling"); impl_->notification.armed = true; } } - // Arm first, then recheck packet/software queues and submit the poll. A - // packet arriving in either gap leaves a readable fd or a MSG_RING CQE. - bool ok = true; - if (!Poll() && rte_ring_empty(lane.rx) && rte_ring_empty(lane.tx) && - impl_->completions.empty()) { - const auto submitted = kernel_.Submit(); - if (!submitted.ok()) { - // Temporary enter backpressure needs another owner-worker round. Leave - // the interrupt cleanup below intact and avoid parking on deferred work. - ok = absl::IsUnavailable(submitted); - } else { - const auto now = NowNs(), deadline = bycorf_bsd_deadline_ns(); - if (deadline != UINT64_MAX) { - const auto delay = deadline <= now - ? 0 - : std::min( - (deadline - now) / 1'000'000, INT_MAX); - timeout_ms = timeout_ms < 0 - ? delay - : std::min(timeout_ms, static_cast(delay)); - } - // A sub-millisecond timer is already due for the millisecond wait API. - // Let the next worker round service it without a zero-timeout syscall. - if (timeout_ms != 0) { - ++impl_->waits; - ok = kernel_.Wait(timeout_ms); - } - } + return true; +} + +bool DpdkBackend::CanWait() const { + const Lane& lane = fabric.lanes[impl_->id]; + return impl_->wait_prepared && impl_->rx_can_wait && + rte_ring_empty(lane.rx) && rte_ring_empty(lane.tx) && + impl_->completions.empty(); +} + +int DpdkBackend::WaitTimeout(int timeout_ms) const { + const auto now = NowNs(), deadline = bycorf_bsd_deadline_ns(); + if (deadline != UINT64_MAX) { + const auto delay = + deadline <= now + ? 0 + : std::min((deadline - now) / 1'000'000, INT_MAX); + timeout_ms = timeout_ms < 0 ? static_cast(delay) + : std::min(timeout_ms, static_cast(delay)); } - lane.parked.store(false, std::memory_order_release); - if (enabled) { - const int rc = DisableRxInterrupt(id); - if (rc) { - std::fprintf( - stderr, "DPDK worker %u: %s\n", id, - AdaptiveError("disable RX interrupt", rc).ToString().c_str()); - return false; - } + return timeout_ms; +} + +absl::Status DpdkBackend::FinishWait(bool waited) { + if (waited) ++impl_->waits; + if (impl_->wait_prepared) { + fabric.lanes[impl_->id].parked.store(false, std::memory_order_release); + impl_->wait_prepared = false; } - return ok && impl_->rx_can_wait; + if (impl_->rx_interrupt_enabled) { + impl_->rx_interrupt_enabled = false; + const int rc = DisableRxInterrupt(impl_->id); + if (rc) return AdaptiveError("disable RX interrupt", rc); + } + if (!impl_->rx_can_wait) + return absl::InternalError("DPDK RX notification is unavailable"); + return absl::OkStatus(); } absl::Status DpdkBackend::SubmitSend(const RegisteredFile& file, diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index 5375a0b..4424743 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -36,6 +36,7 @@ #include "absl/base/internal/sysinfo.h" #endif #include "spdlog/spdlog.h" +#include "worker_wait.h" namespace bycorf { @@ -1065,6 +1066,7 @@ bool Worker::BusyPoll() { CycleNow() + static_cast(busy_poll_cycles_); do { bool did_work = DrainCrossCore(); + did_work |= KernelBackend().Poll(); did_work |= Network::Poll(); did_work |= PollStorage(); if (did_work) { @@ -1122,7 +1124,8 @@ bool Worker::PollStorage() { void Worker::Flush() { FlushWakes(); - const auto status = Network::Submit(); + Network::Flush(); + const auto status = KernelBackend().Submit(); if (!status.ok()) [[unlikely]] { // CQ backpressure and interrupted enters are retried by the next round. // A permanent error cannot make progress by waiting for queue capacity. @@ -1132,6 +1135,12 @@ void Worker::Flush() { } } +bool Worker::HasRunnableWork() const noexcept { + return !ready_.empty() || !next_ready_.empty() || + !foreground_remote_work_.empty() || !background_ready_.empty() || + !next_background_ready_.empty() || !background_remote_work_.empty(); +} + bool Worker::RunOnce(bool wait_for_completion) { if (!initialized_ || submit_failed_) [[unlikely]] { return false; @@ -1142,11 +1151,9 @@ bool Worker::RunOnce(bool wait_for_completion) { // 1. Poll external work once, then run online coroutines within their normal // budget. Deferred Yield continuations are appended after fresh I/O and // cross-core work so yielding really returns control to the event loop. - bool did_work = - !ready_.empty() || !next_ready_.empty() || - !foreground_remote_work_.empty() || !background_ready_.empty() || - !next_background_ready_.empty() || !background_remote_work_.empty(); + bool did_work = HasRunnableWork(); did_work |= DrainCrossCore(); + did_work |= KernelBackend().Poll(); did_work |= Network::Poll(); // This extra foreground slice exists to get ahead of storage completions. // Without pending storage work, the normal slice below already serves the @@ -1247,30 +1254,9 @@ bool Worker::RunOnce(bool wait_for_completion) { return true; } - // 4. Idle — park. Check idle timeouts only here, off the hot path. The - // wake_seq - // handshake closes the lost-wakeup race: snapshot it, recheck the mailbox - // once, then CAS to the parked sentinel. The recheck catches work already - // enqueued; the CAS catches work published after the snapshot (a - // producer's fetch_add changes wake_seq, so the CAS fails and we loop - // instead of park). - CheckIdleConnections(); - WorkerMailbox& mb = cross_core_->mailbox(id_); - const std::uint32_t seq = mb.wake_seq_.load(std::memory_order_acquire); - if (DrainCrossCore()) { - return true; // raced: work arrived; next iteration drains + submits it - } - std::uint32_t expected = seq; - if (!mb.wake_seq_.compare_exchange_strong(expected, kWakeSeqParked, - std::memory_order_acq_rel, - std::memory_order_relaxed)) { - return true; // a producer published work; loop again instead of parking - } - - const int timeout_ms = options_.idle_timeout_ms_ > 0 ? 100 : -1; - const bool ok = Network::Wait(timeout_ms); // blocks; dispatches on wake - mb.wake_seq_.store(0, std::memory_order_release); // leave the parked state - return ok; // next iteration drains what Wait dispatched + // Tests inject races through the observer; optimized builds eliminate this + // empty production callback and its calls in the templated wait protocol. + return WaitForWork([](WaitPhase) {}); } void Worker::Run() { diff --git a/src/runtime/worker_wait.h b/src/runtime/worker_wait.h new file mode 100644 index 0000000..939ebab --- /dev/null +++ b/src/runtime/worker_wait.h @@ -0,0 +1,85 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "bycorf/runtime/worker.h" +#include "spdlog/spdlog.h" + +namespace bycorf { + +template +bool Worker::WaitForWork(Observer&& observe) { + CheckIdleConnections(); + WorkerMailbox& mb = cross_core_->mailbox(id_); + bool parked = false; + bool waited = false; + const auto prepared = Network::PrepareWait(); + // Keep all exits inside this attempt so even partial preparation is cleaned + // up. FinishWait errors participate in the normal worker failure path. + const bool ok = [&] { + if (!prepared.ok()) { + spdlog::error("worker prepare wait failed: {}", + prepared.status().message()); + return false; + } + if (!*prepared) return true; + observe(WaitPhase::kPrepared); + + // Snapshot BEFORE draining. Publication after the empty check changes the + // sequence and defeats CAS; publication after CAS sends a persistent CQE. + const auto seq = mb.wake_seq_.load(std::memory_order_acquire); + bool work = KernelBackend().Poll(); + work |= Network::Poll(); + work |= PollStorage(); + work |= DrainCrossCore(); + observe(WaitPhase::kRechecked); + if (work || HasRunnableWork() || NotifyWake() || Storage::NeedsPolling() || + !Network::CanWait()) + return true; + + auto expected = seq; + if (!mb.wake_seq_.compare_exchange_strong(expected, kWakeSeqParked, + std::memory_order_acq_rel, + std::memory_order_relaxed)) + return true; + parked = true; + observe(WaitPhase::kParked); + + // Preparation and rechecking may queue polls, output, and peer wakes. + // Accepted requests are not necessarily submitted: never sleep through + // temporary enter backpressure or work left in the submission FIFO. + FlushWakes(); + Network::Flush(); + const auto submitted = KernelBackend().Submit(); + if (!submitted.ok()) { + if (absl::IsUnavailable(submitted)) return true; + submit_failed_ = true; + spdlog::error("worker submit before wait failed: {}", + submitted.message()); + return false; + } + observe(WaitPhase::kSubmitted); + if (KernelBackend().HasPending() || HasRunnableWork() || NotifyWake() || + Storage::NeedsPolling() || !Network::CanWait()) + return true; + + const int timeout_ms = + Network::WaitTimeout(options_.idle_timeout_ms_ > 0 ? 100 : -1); + // BSD deadlines below the millisecond wait resolution are serviced by + // the next worker round without issuing a zero-timeout wait syscall. + if (timeout_ms == 0) return true; + observe(WaitPhase::kWaiting); + waited = true; + return KernelBackend().Wait(timeout_ms); + }(); + if (parked) mb.wake_seq_.store(0, std::memory_order_release); + const auto finished = Network::FinishWait(waited); + if (!finished.ok()) { + spdlog::error("worker finish wait failed: {}", finished.message()); + return false; + } + return ok; +} + +} // namespace bycorf diff --git a/tests/dpdk_adaptive_test.cpp b/tests/dpdk_adaptive_test.cpp index a013a9b..1d2591f 100644 --- a/tests/dpdk_adaptive_test.cpp +++ b/tests/dpdk_adaptive_test.cpp @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include @@ -29,6 +31,8 @@ #include #include "bycorf/net/server.h" +#include "bycorf/runtime/foreign_executor.h" +#include "worker_wait_test_peer.h" namespace { std::string_view scenario; @@ -37,12 +41,103 @@ std::atomic enables{0}; std::atomic disables{0}; std::atomic configured{0}; std::atomic injected{false}; +std::atomic race_done{false}; +std::atomic deadlines{0}; + +void CheckWaitRace(bycorf::Worker& worker) { + using Peer = bycorf::WorkerWaitTestPeer; + using Phase = Peer::Phase; + if (scenario == "race-facade") { + bycorf::CheckNetworkRingIsolation(worker); + injected = true; + race_done.store(true, std::memory_order_release); + return; + } + bool reached = false, waiting = false; + unsigned delivered = 0; + const auto boundary = + scenario == "race-notification" ? Phase::kWaiting + : scenario == "race-stop" ? Phase::kPrepared + : (scenario == "race-submit-queue" || scenario == "race-deadline") + ? Phase::kSubmitted + : Phase::kRechecked; + const char* queue_name = + scenario == "race-tx-queue" ? "bycorf_tx_0" : "bycorf_rx_0"; + // Other workers may still be attaching their VNET. Retry only until this + // worker can actually prepare notifications; no timeout rescues its wait. + const auto limit = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!reached && std::chrono::steady_clock::now() < limit) { + const auto before = enables.load(); + EXPECT_TRUE(Peer::Wait(worker, [&](Phase phase) { + waiting |= phase == Phase::kWaiting; + if (phase != boundary) return; + reached = true; + if (scenario == "race-notification") { + const std::uint64_t one = 1; + EXPECT_EQ(write(notification_fd, &one, sizeof(one)), sizeof(one)); + } else if (scenario == "race-stop") { + worker.RequestStop(); + } else if (scenario == "race-mailbox") { + bycorf::detail::ForeignExecutorState ingress{ + .cross_core_ = bycorf::ThisWorker().cross_core_, .worker_id_ = 0}; + std::thread producer([&] { + EXPECT_TRUE(bycorf::detail::PostForeignNotification( + &ingress, {.context_ = &delivered, + .value_ = 0, + .run_fn_ = +[](void* p, std::uint64_t) noexcept { + ++*static_cast(p); + }})); + }); + producer.join(); + } else if (scenario != "race-deadline") { + // Zero-length input is deliberately dropped by Input, but still + // exercises the real software RX queue and mbuf ownership. + auto* packet = rte_pktmbuf_alloc(rte_mempool_lookup("bycorf_packets")); + EXPECT_NE(packet, nullptr); + if (packet) { + auto address = reinterpret_cast(packet); + const int rc = rte_ring_enqueue_elem(rte_ring_lookup(queue_name), + &address, sizeof(address)); + EXPECT_EQ(rc, 0); + if (rc) rte_pktmbuf_free(packet); + } + } + })); + if (reached) { + EXPECT_GT(enables.load(), before); + EXPECT_EQ(enables.load(), disables.load()); + EXPECT_FALSE(Peer::CanWait(worker)); + } + } + EXPECT_TRUE(reached); + EXPECT_EQ(waiting, scenario == "race-notification"); + if (scenario == "race-mailbox") { + EXPECT_TRUE(Peer::Drain(worker)); + EXPECT_EQ(delivered, 1U); + } + if (scenario == "race-notification") { + std::uint64_t counter; + EXPECT_EQ(read(notification_fd, &counter, sizeof(counter)), -1); + EXPECT_EQ(errno, EAGAIN); // The actual notification CQE was dispatched. + } + if (scenario == "race-deadline") EXPECT_GT(deadlines.load(), 0U); + if (scenario == "race-queue" || scenario == "race-submit-queue" || + scenario == "race-tx-queue") { + EXPECT_FALSE(rte_ring_empty(rte_ring_lookup(queue_name))); + EXPECT_TRUE(Peer::Poll(worker)); + EXPECT_TRUE(rte_ring_empty(rte_ring_lookup(queue_name))); + } + injected = reached; + race_done.store(true, std::memory_order_release); +} class IdleService final : public bycorf::Service { public: void Prepare(unsigned) override {} - bycorf::Task Run(bycorf::Worker&, + bycorf::Task Run(bycorf::Worker& worker, bycorf::ServiceContext) override { + if (scenario.starts_with("race-") && worker.id() == 0) + CheckWaitRace(worker); co_return absl::OkStatus(); } void Stop() noexcept override {} @@ -100,10 +195,17 @@ extern "C" int __wrap_rte_eth_dev_rx_intr_disable(uint16_t, uint16_t) { return 0; } +extern "C" std::uint64_t __real_bycorf_bsd_deadline_ns(); +extern "C" std::uint64_t __wrap_bycorf_bsd_deadline_ns() { + ++deadlines; + return scenario == "race-deadline" ? 0 : __real_bycorf_bsd_deadline_ns(); +} + namespace { int RunScenario(std::string_view selected) { scenario = selected; const bool poll = scenario == "poll"; + const bool race = scenario.starts_with("race-"); const bool mlx5 = scenario == "mlx5-empty"; const bool runtime = scenario.starts_with("runtime-"); // Ignore inherited hardware configuration; each process gets its own EAL. @@ -123,7 +225,7 @@ int RunScenario(std::string_view selected) { bycorf::Server server; server.AddService(&service); const auto status = server.Start({.thread_count_ = 2, .pin_workers_ = false}); - if (!poll && !runtime && !mlx5) { + if (!poll && !runtime && !mlx5 && !race) { if (status.ok() || !injected || configured != 1 || status.message().find("BYCORF_DPDK_MODE=poll") == std::string_view::npos) { @@ -137,11 +239,21 @@ int RunScenario(std::string_view selected) { return 1; } if (mlx5) std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if (poll || mlx5) server.RequestStop(); + if (race) { + const auto limit = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!race_done.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < limit) + std::this_thread::yield(); + } + if (poll || mlx5 || race) server.RequestStop(); // A CTest timeout catches peers stranded at the all-worker barriers. server.WaitUntilStopped(); if (poll && (server.exit_code() != 0 || enables != 0)) return 1; if (mlx5 && (server.exit_code() != 0 || enables < 2 || !injected)) return 1; + if (race && + (server.exit_code() != 0 || !injected || testing::Test::HasFailure())) + return 1; if (runtime && (server.exit_code() == 0 || !injected)) return 1; } close(notification_fd); @@ -161,7 +273,11 @@ TEST_P(DpdkAdaptiveDeathTest, HandlesInterruptFailure) { INSTANTIATE_TEST_SUITE_P(Scenarios, DpdkAdaptiveDeathTest, testing::Values("configure", "descriptor", "enable", "disable", "poll", "runtime-enable", - "runtime-disable", "mlx5-empty"), + "runtime-disable", "mlx5-empty", + "race-queue", "race-tx-queue", + "race-submit-queue", "race-mailbox", + "race-notification", "race-stop", + "race-deadline", "race-facade"), [](const testing::TestParamInfo& info) { std::string name = info.param; for (auto& character : name) diff --git a/tests/io_uring_backpressure_test.cpp b/tests/io_uring_backpressure_test.cpp index bb593ec..eacf08a 100644 --- a/tests/io_uring_backpressure_test.cpp +++ b/tests/io_uring_backpressure_test.cpp @@ -459,9 +459,9 @@ TEST(IoUringBackpressure, FixedBufferDescriptorLifetime) { ASSERT_EQ(closed.result, 0); } -// Both facades must submit to the same ring: the network event loop also -// delivers storage completions, and stopping either facade must leave the -// worker-owned ring available until the worker shuts it down. +// Both facades enqueue operations on the same ring. Only the owner advances +// kernel submissions/completions; stopping either facade leaves the ring +// available until the worker shuts it down. TEST(IoUringBackpressure, NetworkAndStorageShareRing) { struct BackendOwner : bycorf::IoUringBackend, bycorf::NetBackend, @@ -487,7 +487,7 @@ TEST(IoUringBackpressure, NetworkAndStorageShareRing) { Completion opened; ASSERT_TRUE( storage.SubmitOpenDirect("/dev/zero", O_RDONLY, 0, {0}, &opened).ok()); - Drain(network, [&] { return opened.calls != 0; }); + Drain(kernel, [&] { return opened.calls != 0; }); ASSERT_EQ(opened.result, 0); int sockets[2]; @@ -504,7 +504,7 @@ TEST(IoUringBackpressure, NetworkAndStorageShareRing) { ASSERT_EQ(sent.calls, 0u); // Kernel storage never requests a separate poll or keeps the worker spinning. ASSERT_FALSE(storage.NeedsPolling()); - Drain(network, [&] { return read.calls != 0 && sent.calls != 0; }); + Drain(kernel, [&] { return read.calls != 0 && sent.calls != 0; }); ASSERT_EQ(read.result, bytes.size()); ASSERT_EQ(sent.result, 1); EXPECT_EQ(bytes, (std::array{})); diff --git a/tests/spdk_backpressure_stub_test.cpp b/tests/spdk_backpressure_stub_test.cpp index 7f34839..fd2c587 100644 --- a/tests/spdk_backpressure_stub_test.cpp +++ b/tests/spdk_backpressure_stub_test.cpp @@ -37,6 +37,7 @@ #include "bycorf/io/spdk_storage.h" #include "bycorf/runtime/worker.h" +#include "worker_wait_test_peer.h" namespace { // Stub C callbacks cannot return GoogleTest fatal assertions or unwind. @@ -265,6 +266,13 @@ void CheckQueues(bycorf::Worker& worker, std::span buffer) { CHECK(first.accepted.size() == 1 && second.accepted.size() == 1); CHECK(first.accepted[0].context == &read); CHECK(second.accepted[0].context == &other); + // Outstanding device I/O has no kernel wake source. Even the final idle + // recheck must veto waiting while accepted and deferred commands remain. + bool waited = false; + CHECK(bycorf::WorkerWaitTestPeer::Wait(worker, [&](auto phase) { + waited |= phase == bycorf::WorkerWaitTestPeer::Phase::kWaiting; + })); + CHECK(!waited); unsigned attempts = first.attempts; for (unsigned i = 0; i < 3; ++i) worker.RunOnce(false); CHECK(first.attempts > attempts); diff --git a/tests/worker_wait_test.cpp b/tests/worker_wait_test.cpp new file mode 100644 index 0000000..2e751f7 --- /dev/null +++ b/tests/worker_wait_test.cpp @@ -0,0 +1,193 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include + +#include "bycorf/runtime/foreign_executor.h" +#include "worker_wait_test_peer.h" + +namespace bycorf { +namespace { +using Phase = WorkerWaitTestPeer::Phase; + +class WorkerWait : public testing::Test { + protected: + void SetUp() override { + for (unsigned id = 0; id < 2; ++id) { + cc.mailbox(id).wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(cc.mailbox(id).wake_fd_, 0); + workers[id].BindCrossCore(id, &cc); + SetThisWorker(id, &cc, &workers[id]); + const auto status = workers[id].Init({.recv_buffer_count_ = 0}); + ASSERT_TRUE(status.ok()) << status; + WorkerWaitTestPeer::Flush(workers[id]); + } + SetThisWorker(0, &cc, &workers[0]); + // Lost wakes fail the case instead of being rescued by a periodic timer. + alarm(5); + } + void TearDown() override { + alarm(0); + for (unsigned id = 0; id < 2; ++id) { + SetThisWorker(id, &cc, &workers[id]); + workers[id].Shutdown(); + workers[id].DestroyDetachedTasks(); + close(cc.mailbox(id).wake_fd_); + } + MutableThisWorker() = {}; + } + RemoteNotification Notification() { + return {.context_ = &calls, + .value_ = 0, + .run_fn_ = +[](void* context, std::uint64_t) noexcept { + ++*static_cast(context); + }}; + } + void PostPeer() { + SetThisWorker(1, &cc, &workers[1]); + PostNotification(&cc, 0, Notification()); + WorkerWaitTestPeer::Flush(workers[1]); + SetThisWorker(0, &cc, &workers[0]); + } + void PostForeign() { + detail::ForeignExecutorState state{.cross_core_ = &cc, .worker_id_ = 0}; + std::thread producer([&] { + EXPECT_TRUE(detail::PostForeignNotification(&state, Notification())); + }); + producer.join(); + } + CrossCore cc{2}; + Worker workers[2]; + unsigned calls = 0; +}; + +TEST_F(WorkerWait, BusyPollAdvancesSharedRing) { + struct Completion : IoCompletion { + unsigned calls = 0; + void Complete(Worker&, int result, unsigned) override { + ++calls; + EXPECT_EQ(result, -ECANCELED); + } + } completion; + auto& kernel = WorkerWaitTestPeer::Kernel(workers[0]); + const __kernel_timespec timeout{.tv_sec = 60, .tv_nsec = 0}; + ASSERT_TRUE(kernel.SubmitTimeout(timeout, &completion).ok()); + ASSERT_TRUE(kernel.SubmitCancel(&completion).ok()); + EXPECT_TRUE(WorkerWaitTestPeer::BusyPoll(workers[0])); + EXPECT_EQ(completion.calls, 1U); + workers[0].Shutdown(); +} + +TEST_F(WorkerWait, NetworkFacadeDoesNotAdvanceSharedRing) { + CheckNetworkRingIsolation(workers[0]); +} + +TEST_F(WorkerWait, MailboxPublicationAfterEmptyCheckDefeatsPark) { + for (bool foreign : {false, true}) { + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == Phase::kRechecked) { + if (foreign) + PostForeign(); + else + PostPeer(); + } + waiting |= phase == Phase::kWaiting; + })); + EXPECT_FALSE(waiting); + EXPECT_NE(cc.mailbox(0).wake_seq_.load(), kWakeSeqParked); + EXPECT_TRUE(WorkerWaitTestPeer::Drain(workers[0])); + } + EXPECT_EQ(calls, 2U); +} + +TEST_F(WorkerWait, PublicationAfterParkLeavesPersistentWake) { + for (auto boundary : {Phase::kParked, Phase::kWaiting}) { + for (bool foreign : {false, true}) { + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == boundary) { + if (foreign) + PostForeign(); + else + PostPeer(); + } + waiting |= phase == Phase::kWaiting; + })); + EXPECT_TRUE(waiting); + EXPECT_EQ(cc.mailbox(0).wake_seq_.load(), 0U); + EXPECT_TRUE(WorkerWaitTestPeer::Drain(workers[0])); + } + } + EXPECT_EQ(calls, 4U); +} + +TEST_F(WorkerWait, StopConsumedDuringRecheckDoesNotSleepAgain) { + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == Phase::kPrepared) workers[0].RequestStop(); + waiting |= phase == Phase::kWaiting; + })); + EXPECT_FALSE(waiting); + EXPECT_TRUE(workers[0].NotifyWake()); + EXPECT_NE(cc.mailbox(0).wake_seq_.load(), kWakeSeqParked); +} + +TEST_F(WorkerWait, StopAfterFinalCheckWakesInfiniteWait) { + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == Phase::kWaiting) { + waiting = true; + std::thread producer([&] { workers[0].RequestStop(); }); + producer.join(); + } + })); + EXPECT_TRUE(waiting); + EXPECT_TRUE(workers[0].NotifyWake()); + EXPECT_EQ(cc.mailbox(0).wake_seq_.load(), 0U); +} + +Task CountTask(unsigned& count) { + ++count; + co_return absl::OkStatus(); +} + +TEST_F(WorkerWait, RecheckAndSubmissionCannotStrandRunnableTasks) { + for (auto boundary : {Phase::kRechecked, Phase::kSubmitted}) { + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == boundary) workers[0].Spawn(CountTask(calls)); + waiting |= phase == Phase::kWaiting; + })); + EXPECT_FALSE(waiting); + EXPECT_NE(cc.mailbox(0).wake_seq_.load(), kWakeSeqParked); + EXPECT_TRUE(workers[0].RunOnce(false)); + } + EXPECT_EQ(calls, 2U); +} + +TEST_F(WorkerWait, UnsubmittedOperationVetoesFinalWait) { + struct Completion : IoCompletion { + void Complete(Worker&, int, unsigned) override { + ADD_FAILURE() << "unsubmitted timeout must not complete"; + } + } completion; + bool waiting = false; + ASSERT_TRUE(WorkerWaitTestPeer::Wait(workers[0], [&](Phase phase) { + if (phase == Phase::kSubmitted) { + const __kernel_timespec timeout{.tv_sec = 60, .tv_nsec = 0}; + EXPECT_TRUE(workers[0].SubmitTimeout(timeout, &completion).ok()); + } + waiting |= phase == Phase::kWaiting; + })); + EXPECT_FALSE(waiting); + EXPECT_EQ(cc.mailbox(0).wake_seq_.load(), 0U); + // Quiesce the queued operation while its completion tag is still alive. + workers[0].Shutdown(); +} +} // namespace +} // namespace bycorf diff --git a/tests/worker_wait_test_peer.h b/tests/worker_wait_test_peer.h new file mode 100644 index 0000000..048ef20 --- /dev/null +++ b/tests/worker_wait_test_peer.h @@ -0,0 +1,58 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include "../src/runtime/worker_wait.h" + +namespace bycorf { +class WorkerWaitTestPeer { + public: + using Phase = Worker::WaitPhase; + template + static bool Wait(Worker& worker, Observer&& observer) { + return worker.WaitForWork(std::forward(observer)); + } + static void Flush(Worker& worker) { worker.Flush(); } + static bool Drain(Worker& worker) { return worker.DrainCrossCore(); } + static bool Poll(Worker& worker) { return worker.Network::Poll(); } + static void FlushNetwork(Worker& worker) { worker.Network::Flush(); } + static IoUringBackend& Kernel(Worker& worker) { + return worker.KernelBackend(); + } + static bool BusyPoll(Worker& worker) { + worker.options_.busy_poll_us_ = 1; + worker.busy_poll_cycles_ = 1; + return worker.BusyPoll(); + } + static bool CanWait(Worker& worker) { return worker.Network::CanWait(); } +}; +// Cancellation of an unsubmitted request produces a deferred software +// completion. This gives a deterministic ring event without timing sleeps or +// a real NIC, and checks both kernel-TCP and DPDK facade isolation. +inline void CheckNetworkRingIsolation(Worker& worker) { + struct Completion : IoCompletion { + unsigned calls = 0; + void Complete(Worker&, int result, unsigned) override { + ++calls; + EXPECT_EQ(result, -ECANCELED); + } + } completion; + auto& kernel = WorkerWaitTestPeer::Kernel(worker); + const __kernel_timespec timeout{.tv_sec = 60, .tv_nsec = 0}; + ASSERT_TRUE(kernel.SubmitTimeout(timeout, &completion).ok()); + WorkerWaitTestPeer::FlushNetwork(worker); + EXPECT_TRUE(kernel.HasPending()) << "network flush submitted the shared ring"; + ASSERT_TRUE(kernel.SubmitCancel(&completion).ok()); + (void)WorkerWaitTestPeer::Poll(worker); + EXPECT_EQ(completion.calls, 0U) << "network poll dispatched ring work"; + // Drive terminal cancellation before releasing its tag even on regression. + while (completion.calls == 0) { + ASSERT_TRUE(kernel.Submit().ok()); + kernel.Poll(); + } + EXPECT_EQ(completion.calls, 1U); +} +} // namespace bycorf