diff --git a/CMakeLists.txt b/CMakeLists.txt index 97f9cdb..d587b8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,7 +186,8 @@ if (BYCORF_ENABLE_RDMA) find_package(PkgConfig REQUIRED) pkg_check_modules(BYCORF_IBVERBS REQUIRED IMPORTED_TARGET libibverbs) pkg_check_modules(BYCORF_RDMACM REQUIRED IMPORTED_TARGET librdmacm) - target_sources(bycorf PRIVATE src/io/rdma_memory.cpp) + target_sources(bycorf PRIVATE src/io/rdma_memory.cpp src/rdma/rc.cpp + src/rdma/rc_engine.cpp) target_link_libraries(bycorf PUBLIC PkgConfig::BYCORF_IBVERBS PkgConfig::BYCORF_RDMACM) endif() @@ -473,6 +474,7 @@ if (BYCORF_BUILD_TESTS) tests/cross_core_drain_test.cpp tests/worker_scheduler_test.cpp tests/received_buffer_queue_test.cpp + tests/local_shared_ptr_test.cpp tests/backend_selection_test.cpp) target_link_libraries(bycorf_unit_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_unit_tests 30 "runtime;unit") @@ -487,6 +489,25 @@ if (BYCORF_BUILD_TESTS) endforeach() bycorf_discover_tests(bycorf_rdma_memory_tests 30 "runtime;rdma;unit") + add_executable(bycorf_rdma_rc_fake_tests tests/rdma_rc_test.cpp tests/rdma_fake.cpp) + target_compile_definitions(bycorf_rdma_rc_fake_tests PRIVATE BYCORF_RDMA_FAKE=1) + target_link_libraries(bycorf_rdma_rc_fake_tests PRIVATE bycorf::core GTest::gtest_main) + foreach(symbol IN ITEMS rdma_create_event_channel rdma_destroy_event_channel + rdma_create_id rdma_destroy_id rdma_bind_addr rdma_listen rdma_resolve_addr + rdma_resolve_route rdma_connect rdma_accept rdma_establish rdma_disconnect + rdma_reject rdma_get_cm_event rdma_ack_cm_event rdma_init_qp_attr + ibv_get_device_name ibv_open_device ibv_close_device ibv_query_device + ibv_alloc_pd ibv_dealloc_pd ibv_create_comp_channel ibv_destroy_comp_channel + ibv_create_cq ibv_destroy_cq ibv_create_qp ibv_modify_qp ibv_destroy_qp + ibv_get_cq_event ibv_ack_cq_events ibv_get_async_event ibv_ack_async_event) + target_link_options(bycorf_rdma_rc_fake_tests PRIVATE "LINKER:--wrap=${symbol}") + endforeach() + bycorf_discover_tests(bycorf_rdma_rc_fake_tests 30 "runtime;rdma;stub") + + add_executable(bycorf_rdma_rc_tests tests/rdma_rc_test.cpp) + target_link_libraries(bycorf_rdma_rc_tests PRIVATE bycorf::core GTest::gtest_main) + bycorf_discover_tests(bycorf_rdma_rc_tests 30 "runtime;rdma;integration") + add_executable(bycorf_rdma_registration_tests tests/rdma_registration_test.cpp) target_link_libraries(bycorf_rdma_registration_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_rdma_registration_tests 30 "runtime;rdma;integration") diff --git a/README.md b/README.md index 3deb16e..c12bfcd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ connections, and asynchronous I/O; applications provide services above - [Name](#name) - [Build and test](#build-and-test) -- [Optional RDMA registration](#optional-rdma-registration) +- [Optional RDMA connections and registration](#optional-rdma-connections-and-registration) - [Run the examples](#run-the-examples) - [Use Bycorf in a CMake project](#use-bycorf-in-a-cmake-project) - [Development](#development) @@ -179,10 +179,10 @@ Follow the [DPDK/SPDK build and test guide](docs/dpdk-prototype.md) for its additional dependencies, virtual-device checks, physical-device configuration, and experimental compatibility limits. -## Optional RDMA registration +## Optional RDMA connections and registration -`BYCORF_ENABLE_RDMA` enables the private worker facade and explicit memory -registration API. It defaults to off and is independent of +`BYCORF_ENABLE_RDMA` enables the private worker facade, RC connection API and +explicit memory registration. It defaults to off and is independent of `BYCORF_KERNEL_BYPASS`; enabling it requires libibverbs and librdmacm development packages, without DPDK/SPDK. On Ubuntu: @@ -190,7 +190,7 @@ packages, without DPDK/SPDK. On Ubuntu: sudo apt-get install --no-install-recommends libibverbs-dev librdmacm-dev cmake -S . -B build-rdma -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBYCORF_ENABLE_RDMA=ON cmake --build build-rdma --parallel -ctest --test-dir build-rdma -R '^bycorf_RdmaMemory\.' --output-on-failure +ctest --test-dir build-rdma -R '^bycorf_(RdmaMemory|RcLoopFake)\.' --output-on-failure ``` The device-free `RdmaMemory` suite injects verbs failures and verifies lazy @@ -206,9 +206,21 @@ BYCORF_RDMA_TEST_DEVICE=rxe0 ctest --test-dir build-rdma \ This opens the selected device, registers and deregisters memory, creates real QPs sharing a PD, and checks permissions, bounds, and mismatched PD rejection. -It requires access to the device's uverbs node and sufficient memlock. It does -not test a connected data-transfer or immediate-data protocol. No device is -opened by unused workers or test discovery. +It requires access to the device's uverbs node and sufficient memlock. For RC +connection tests on an existing Soft-RoCE interface, supply its local IP: + +```bash +BYCORF_RDMA_TEST_IP=192.0.2.1 ctest --test-dir build-rdma \ + -R '^bycorf_RcLoop\.' --output-on-failure +``` + +These exercise idle-worker connection events, private data, distinct worker +contexts, shared CQ/PD ownership, refused connections and close/accept races. +The device-free `RcLoopFake` cases additionally inject notification races, +provider capacity rounding, budget exhaustion, deadlines, CQ/device failures, +and late events during cancellation and shutdown. They use real io_uring with +link-time CM/verbs stubs. Data-transfer and immediate-data protocols remain +outside these tests. No device is opened by unused workers or test discovery. Use `Worker::OpenRdmaDevice(name)` or `AcquireRdmaDomain(owning_context)` on the owner worker, then `RegisterRdmaMemory(domain, backing_span, explicit_access)`. @@ -219,6 +231,18 @@ backing memory and must stop remote access before releasing registrations and freeing it. See [RDMA ownership](docs/architecture/rdma.md) for remote descriptors, permissions, CM context ownership, and the independent storage-buffer contract. +The RC API is in `bycorf/rdma/rc.h`. On the owning worker, call +`RcListener::Listen(worker, numeric_ip, port)` and await `listener.Accept()`, or +await `RcConnection::Connect(worker, numeric_ip, port)`. Pass `RcOptions` to set +QP/CQ capacities and the handshake timeout, and an optional byte span for CM +private data. Await `connection.Close()` or `listener.Close()` to join resource +retirement. Destruction only requests close. Use +`connection.protection_domain()` when registering memory for that connection. +An accepted connection stays on the listener's worker; moving a C++ handle +does not move worker ownership. Private data is limited to 56 bytes; CM may +report a padded peer-data length. The first connection fixes that worker/device +CQ capacity, and subsequent connections must request the same capacity. + ## Run the examples The default build enables `BYCORF_BUILD_EXAMPLES`. If it was disabled, set diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 874c894..0038402 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -23,8 +23,9 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging. | Module | Responsibility | Document | |---|---|---| | Runtime | Worker lifecycle, coroutine scheduling, sleep coordination, cross-worker and foreign ingress | [Runtime](runtime.md) | -| I/O | Worker-owned shared io_uring, network/storage adapters, optional RDMA registration and SPDK storage, shared DPDK environment | [I/O backends](io.md), [RDMA ownership](rdma.md) | +| I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | | Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) | +| RDMA | Worker-owned RC connections, CM/QP lifecycle, shared CQ notifications and memory registration | [RDMA](rdma.md) | | RPC | Framed request/reply protocol above streams | [RPC](rpc.md) | Each connection and its coroutines have one worker owner. Key ownership and @@ -38,8 +39,8 @@ worker before that coroutine uses its stream again. Startup selection independently activates network and storage backends and remains fixed for the process lifetime. Both default to io_uring even in a bypass-capable build. -`BYCORF_ENABLE_RDMA` independently enables the RDMA facade and registration -layer; resources are acquired only on explicit use. +`BYCORF_ENABLE_RDMA` independently enables the RDMA facade, RC connections and +registration layer; resources are acquired only on explicit use. The [prototype runbook](../dpdk-prototype.md) describes the DPDK configuration. Architecture documents describe the current core model and stable tradeoffs. diff --git a/docs/architecture/io.md b/docs/architecture/io.md index a4a317a..39f031e 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -44,11 +44,12 @@ Network adapters supply notification preparation, idle checks, deadlines, and cleanup to the [worker sleep protocol](runtime.md#sleep-and-wakeups). The worker enters the ring's blocking wait; adapters never block independently. -The ring outlives all facades. RDMA drops its weak resource cache on shutdown; -registration and operation handles retain their PD/context independently. The -worker initializes the ring first, stops storage and networking in that order, -then shuts the ring down. Storage dispatch remains -valid through shutdown completion draining; SPDK callbacks use the owning +The ring outlives all facades. RDMA stops admission before ring shutdown but +retains channel polls until the ring has quiesced; it then retires remaining +RDMA events and connection resources. Exported registration handles retain +their PD/context independently. The worker initializes the ring first, stops +storage and networking in that order, then shuts the ring down. Storage dispatch +remains valid through shutdown completion draining; SPDK callbacks use the owning worker's TLS to reach its storage adapter. Kernel registrations remain live until ring shutdown quiesces kernel access. diff --git a/docs/architecture/rdma.md b/docs/architecture/rdma.md index 0f17263..6419c4f 100644 --- a/docs/architecture/rdma.md +++ b/docs/architecture/rdma.md @@ -14,14 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. --> -# RDMA registration and ownership +# RDMA connections, notifications and memory ownership `BYCORF_ENABLE_RDMA` adds an independent I/O capability backed by libibverbs and librdmacm. It defaults to off and does not select or require DPDK/SPDK. The private `RdmaBackend` facade reaches the worker's existing io_uring through `KernelBackend()` for channel readiness and cancellation. It creates no ring, polling thread, thread pool, or process-wide blocking lock. The worker -continues to own shared-ring submission and completion dispatch. +continues to own shared-ring submission, completion dispatch and sleeping. +`bycorf::rdma::RcListener` and `RcConnection` provide numeric-address RC +Listen/Accept/Connect/Close and CM private data independently of TCP streams. Construction does not enumerate or open devices. Explicit `OpenRdmaDevice` selects a named verbs device; `AcquireRdmaDomain` accepts an owning reference to @@ -37,8 +39,97 @@ independent of worker shutdown. Partial initialization unwinds in reverse order. PD and retain the domain until QP destruction. A registered MR can be used by multiple QPs in the same PD; it belongs to neither a particular QP nor an IMM (immediate-data) operation. Operation leases check both the QP's context and PD. -Connection management, QP posting and completion protocols are outside this -registration layer. +RC connections expose their actual protection domain for registration. A caller +must use that domain rather than assume a separately opened device handle has +the same PD. Data-posting and application completion protocols are outside the +connection API. + +## RC ownership and connection flow + +Each worker lazily owns one nonblocking CM channel and its ID registry. Each +listener and every connection accepted by it stay on that worker; a connection +returned by Accept survives listener close. Connect and accepted handshakes +have one absolute monotonic deadline covering address resolution, route +resolution and establishment. Submission backpressure does not restart it. +Arguments are copied before returning a lazy task. Private data is limited to +56 bytes in both directions; peer data is copied before acknowledging the CM +event, with the length reported by CM (which can include provider padding). +Listen binds immediately; Accept, Connect and Close are awaitable operations. + +Librdmacm shares its routing contexts process-wide. RC therefore opens a private +verbs context per worker on the device selected by CM, and uses CM's external-QP +mode. CM negotiates the path/QP number; the worker creates only RC QPs in its own +PD and applies CM-provided INIT/RTR/RTS attributes, then completes the active +handshake with `rdma_establish`. It does not replace CM's context or attach its +QP to the CM ID. QP, PD, MR, CQ and the verbs asynchronous event stream belong to +the same private context, with exactly one worker reading that event stream. +Other workers neither consume its events nor migrate its connections. + +Each private context shares one completion channel and fixed-capacity CQ. +Admission reserves the provider's actual send plus receive WR capacities for +each QP, including capacities rounded up by the provider. All send WRs are +signalled. A QP remains unconnected and has no posted work until its actual +budget fits; excess capacity fails admission instead of growing the CQ. +Reservations cover error/flush completions as well as successful completions, +and are returned only after QP destruction and CQ draining. + +## Notification and failure flow + +The CM channel, completion channel and private context's async-event fd are +nonblocking, with persistent one-shot readiness polls in the shared io_uring. +A CQ notification is requested before checking the CQ, including initial +activation. Each delivered CQ event is acknowledged, notification is requested +again, and the CQ is drained before rearming the fd poll. An event may remain +after its CQE was already consumed; it still needs acknowledgement and rearming. +Notifications stay enabled during busy and idle periods. This makes pending +work either visible in the CQ recheck or capable of waking an idle worker; +there is no separate polling thread or second ring. + +CM events are copied and acknowledged before dispatch can destroy an ID. +Failures retain the CM event name/raw status or the failing verbs operation. +QP errors terminate the affected connection; shared CQ errors, device fatal +errors and device removal terminate every connection on that device context. +Port errors terminate connections using the affected port. Event objects are +acknowledged before this fanout can destroy their referenced resources. A +failed CM channel stops all endpoints on its worker and rejects new admission. + +## Close and worker shutdown + +Public handles are move-only and used on their owner worker. Engine, device, +listener and connection states use non-atomic strong/weak ownership. Every +handle and associated coroutine frame is accessed, moved and destroyed on its +creating thread, including any handles retained after worker shutdown; they +cannot be handed to another thread for cleanup. Exported PD/MR/context handles +retain their separate shared ownership contract. Destruction requests close; +awaiting Close joins retirement. Closing stops admission, +disconnects the CM association and puts the external QP into ERR. The +connection retains its timer tag until the timer's terminal completion, even +when cancellation has already been accepted. It leaves the ID registry before +resource destruction and releases its CQ budget only after the QP is destroyed +and the CQ drained. The shared channel polls reference the worker's event +engine/device state, never a listener or connection object. + +Listener close terminates queued and handshaking children and wakes Accept; +already returned connections remain independent. One Accept and one Close +waiter per handle are supported. Waiters detach when their coroutine frames +are reclaimed. Failed device state is released only after its connections and +both fd polls have retired. A cancellation request or the cancel request's own +CQE is not a retirement boundary for the original fd poll. + +Worker shutdown first stops RDMA admission and requests cancellation while +retaining every poll tag and descriptor. It then quiesces the shared ring. +Only after ring shutdown confirms that no kernel operation can refer to those +tags does RDMA retire requests that did not receive callbacks, acknowledge +remaining events, destroy QPs/CM IDs, and release channels/CQs. Coroutine frames +are reclaimed afterward. Exported registration/PD handles retain their own +context independently. `Server` quiesces the ring and reclaims coroutine frames +on the worker thread. Direct `Runtime` integrations must likewise call worker +`Shutdown()` and `DestroyDetachedTasks()` on that thread before it exits, and +release any application-owned RC handles and Tasks there. Unexpected destruction +failures fail closed rather than +release resources that may still be referenced by the provider. + +## Registered memory `RegisteredMemory` registers caller-owned backing memory with explicit access permissions. Copies, registered slices, and operation leases retain the same @@ -67,4 +158,6 @@ access and lifetime requirements. Sources: `include/bycorf/io/rdma_backend.h`, `include/bycorf/io/rdma_memory.h`, `src/io/rdma_memory.cpp`, -`include/bycorf/runtime/worker.h`, `src/runtime/worker.cpp`. +`include/bycorf/rdma/rc.h`, `src/rdma/rc.cpp`, `src/rdma/rc_engine.cpp`, +`src/rdma/rc_internal.h`, `include/bycorf/runtime/worker.h`, +`src/runtime/worker.cpp`, `src/io/io_uring_backend.cpp`. diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index ad6ac73..c4edb18 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -53,7 +53,11 @@ notifications, rechecks I/O, mailbox and scheduler work, completes the mailbox wake-sequence handshake, and submits operations and peer wakes. After a final idle check it waits on the shared io_uring. Runnable work, a stop request, unsubmitted operations, or storage requiring polling prevent sleep. Network -deadlines and enabled connection-idle checks bound the wait. +deadlines and enabled connection-idle checks bound the wait. Optional RDMA CM, +CQ-notification and device-async polls are persistent wake sources on this same +ring; RDMA connection deadlines are absolute ring timeouts. CQ notifications +are always armed and rechecked, so RDMA needs no busy polling or additional +sleep-preparation phase. Cross-worker submissions enter destination mailboxes and resume coroutines on their original workers. The mailbox handshake makes racing publication either @@ -78,8 +82,11 @@ an empty selection means every worker. Prepare sees the full runtime count, while Run and FinalizeWorker execute only on selected workers. During shutdown, workers first leave their loops, then quiesce I/O and reclaim coroutine frames on their owning native threads. Service finalizers run after all worker frames -are reclaimed. A worker that fails initialization contributes to both teardown -barriers so initialized peers can finish. Temporary I/O submission shortages +are reclaimed. RDMA uses a two-phase shutdown: stop admission before the ring +is quiesced, then release retained polls/events and connection resources after +kernel access has ended. This also covers accepted operations whose callbacks +are suppressed by ring shutdown. A worker that fails initialization contributes +to both teardown barriers so initialized peers can finish. Temporary I/O submission shortages are retried by the worker loop. A permanent submission error, notification preparation or cleanup failure, or shared-ring wait failure stops peer workers before entering those barriers and yields a nonzero exit. Services outlive diff --git a/include/bycorf/base/local_shared_ptr.h b/include/bycorf/base/local_shared_ptr.h new file mode 100644 index 0000000..2748fd4 --- /dev/null +++ b/include/bycorf/base/local_shared_ptr.h @@ -0,0 +1,242 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#ifndef NDEBUG +#include +#endif + +namespace bycorf { +namespace local_shared_detail { +// Same allocating-thread contract as Lavik's LocalSharedPtr. The erased +// control also supports incomplete types and weak references for RC handles. +struct Control { + std::size_t strong = 1; + // One implicit weak reference belongs to the entire strong-owner cohort. + // It keeps the control alive while T's destructor releases its own weak refs. + std::size_t weak = 1; + void (*destroy)(Control*) noexcept; + void (*deallocate)(Control*) noexcept; +#ifndef NDEBUG + const std::thread::id owner = std::this_thread::get_id(); +#endif + void AssertOwner() const noexcept { +#ifndef NDEBUG + assert(owner == std::this_thread::get_id() && + "LocalSharedPtr used outside its owner thread"); +#endif + } + void RetainStrong() noexcept { + AssertOwner(); + assert(strong && strong != std::numeric_limits::max()); + ++strong; + } + void RetainWeak() noexcept { + AssertOwner(); + assert(weak && weak != std::numeric_limits::max()); + ++weak; + } + void ReleaseWeak() noexcept { + AssertOwner(); + if (--weak == 0) deallocate(this); + } + void ReleaseStrong() noexcept { + AssertOwner(); + if (--strong == 0) { + destroy(this); + ReleaseWeak(); + } + } +}; + +template +struct Allocation final : Control { + // Object destruction and allocation release happen at different times when + // weak observers survive. Keeping T in a union avoids double destruction. + union { + T value; + }; + template + explicit Allocation(Args&&... args) + : Control{.destroy = + [](Control* c) noexcept { + std::destroy_at(&static_cast(c)->value); + }, + .deallocate = + [](Control* c) noexcept { + delete static_cast(c); + }}, + value(std::forward(args)...) {} + ~Allocation() {} +}; +} // namespace local_shared_detail + +template +class LocalWeakPtr; +template +class LocalSharedPtr; +template +LocalSharedPtr MakeLocalShared(Args&&... args); + +// Non-atomic shared ownership confined to the allocating thread, including +// access, copying, moving and destruction, even after the worker shuts down. +// Coroutine frames carrying these handles must also be destroyed there. +// Debug builds check thread affinity before accessing the control block. +// Supports incomplete types and T -> const T; no aliasing, raw adoption, +// arrays, custom deleters or cross-thread handoff. Use MakeLocalShared. +template +class LocalSharedPtr { + static_assert(std::is_object_v && !std::is_array_v && + !std::is_volatile_v); + template + static constexpr bool kCompatible = + std::is_same_v, std::remove_const_t> && + std::is_convertible_v; + + public: + constexpr LocalSharedPtr() noexcept = default; + constexpr LocalSharedPtr(std::nullptr_t) noexcept {} + LocalSharedPtr(const LocalSharedPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainStrong(); + } + template + requires kCompatible + LocalSharedPtr(const LocalSharedPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainStrong(); + } + LocalSharedPtr(LocalSharedPtr&& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + other.control_ = nullptr; + other.value_ = nullptr; + } + ~LocalSharedPtr() { reset(); } + LocalSharedPtr& operator=(LocalSharedPtr other) noexcept { + swap(other); + return *this; + } + void reset() noexcept { + auto* control = CheckedControl(); + control_ = nullptr; + value_ = nullptr; + if (control) control->ReleaseStrong(); + } + void swap(LocalSharedPtr& other) noexcept { + (void)CheckedControl(); + (void)other.CheckedControl(); + std::swap(control_, other.control_); + std::swap(value_, other.value_); + } + T* get() const noexcept { + (void)CheckedControl(); + return value_; + } + T* operator->() const noexcept { + assert(value_); + return get(); + } + T& operator*() const noexcept { return *operator->(); } + explicit operator bool() const noexcept { return get() != nullptr; } + std::size_t use_count() const noexcept { + auto* control = CheckedControl(); + return control ? control->strong : 0; + } + friend bool operator==(const LocalSharedPtr& a, + const LocalSharedPtr& b) noexcept { + return a.get() == b.get(); + } + friend bool operator==(const LocalSharedPtr& a, std::nullptr_t) noexcept { + return !a; + } + + private: + template + friend class LocalSharedPtr; + template + friend class LocalWeakPtr; + template + friend LocalSharedPtr MakeLocalShared(Args&&...); + LocalSharedPtr(local_shared_detail::Control* control, T* value) noexcept + : control_(control), value_(value) {} + local_shared_detail::Control* CheckedControl() const noexcept { + if (control_) control_->AssertOwner(); + return control_; + } + local_shared_detail::Control* control_ = nullptr; + T* value_ = nullptr; +}; + +// Non-owning observation with the same thread confinement as LocalSharedPtr. +// lock() retains a live object; after the last strong release it returns empty. +// Weak handles keep only the allocation/control alive, never the object. +template +class LocalWeakPtr { + public: + constexpr LocalWeakPtr() noexcept = default; + LocalWeakPtr(const LocalSharedPtr& strong) noexcept + : control_(strong.CheckedControl()), value_(strong.value_) { + if (control_) control_->RetainWeak(); + } + LocalWeakPtr(const LocalWeakPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainWeak(); + } + LocalWeakPtr(LocalWeakPtr&& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + other.control_ = nullptr; + other.value_ = nullptr; + } + ~LocalWeakPtr() { reset(); } + LocalWeakPtr& operator=(LocalWeakPtr other) noexcept { + swap(other); + return *this; + } + void swap(LocalWeakPtr& other) noexcept { + (void)CheckedControl(); + (void)other.CheckedControl(); + std::swap(control_, other.control_); + std::swap(value_, other.value_); + } + void reset() noexcept { + auto* control = CheckedControl(); + control_ = nullptr; + value_ = nullptr; + if (control) control->ReleaseWeak(); + } + bool expired() const noexcept { + auto* control = CheckedControl(); + return !control || control->strong == 0; + } + LocalSharedPtr lock() const noexcept { + if (expired()) return {}; + control_->RetainStrong(); + return LocalSharedPtr(control_, value_); + } + + private: + local_shared_detail::Control* CheckedControl() const noexcept { + if (control_) control_->AssertOwner(); + return control_; + } + local_shared_detail::Control* control_ = nullptr; + T* value_ = nullptr; +}; + +// Constructs object and control in one allocation. Constructor failure frees +// that allocation; T's destructor must not throw. The last strong release +// destroys T and the last weak release frees the allocation. +template +LocalSharedPtr MakeLocalShared(Args&&... args) { + auto* block = new local_shared_detail::Allocation>( + std::forward(args)...); + return LocalSharedPtr(block, std::addressof(block->value)); +} +} // namespace bycorf diff --git a/include/bycorf/io/io_uring_backend.h b/include/bycorf/io/io_uring_backend.h index 7fe46f0..c71e146 100644 --- a/include/bycorf/io/io_uring_backend.h +++ b/include/bycorf/io/io_uring_backend.h @@ -135,6 +135,10 @@ class IoUringBackend { absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag); absl::Status SubmitTimeout(const __kernel_timespec& timeout, IoCompletion* tag); + // CLOCK_MONOTONIC absolute deadline. Admission/submission backpressure must + // not restart the budget. Caller retains timespec/tag until terminal CQE. + absl::Status SubmitDeadline(const __kernel_timespec& deadline, + IoCompletion* tag); // Observe a host descriptor (e.g. a DPDK RX notification) through the same // CQ as storage and MSG_RING. The caller retains tag until completion/cancel. absl::Status SubmitPoll(int fd, unsigned events, IoCompletion* tag); diff --git a/include/bycorf/io/rdma_backend.h b/include/bycorf/io/rdma_backend.h index cf5519f..54d88ab 100644 --- a/include/bycorf/io/rdma_backend.h +++ b/include/bycorf/io/rdma_backend.h @@ -20,6 +20,7 @@ #if BYCORF_ENABLE_RDMA #include "bycorf/io/io_uring_backend.h" #include "bycorf/io/rdma_memory.h" +#include "bycorf/io/rdma_reactor.h" namespace bycorf { // Private CRTP facade: the owner supplies its existing KernelBackend(). Neither @@ -55,9 +56,28 @@ class RdmaBackend { absl::Status CancelRdmaEventPoll(IoCompletion* tag) { return Kernel().SubmitCancel(tag); } - // Drops only cache entries. Outstanding handles continue to own resources; - // callers must quiesce QPs/remote access before releasing those handles. - void Shutdown() noexcept { resources_.Shutdown(); } + // Absolute CLOCK_MONOTONIC deadline; caller retains deadline/tag until its + // terminal completion, including deferred submission and cancellation. + absl::Status SubmitRdmaDeadline(const __kernel_timespec& deadline, + IoCompletion* tag) { + return Kernel().SubmitDeadline(deadline, tag); + } + // Stop admission before the ring shuts down. Tags/resources remain retained + // until terminal completions or AfterRingShutdown confirms kernel quiescence. + void Shutdown() { rdma::detail::BeginShutdown(engine_); } + void AfterRingShutdown() { + rdma::detail::AfterRingShutdown(engine_); + engine_.reset(); + resources_.Shutdown(); + } + absl::StatusOr> GetRdmaEngine() { + if (!engine_) { + auto created = rdma::detail::CreateEngine(static_cast(*this)); + if (!created.ok()) return created.status(); + engine_ = *created; + } + return engine_; + } private: friend Owner; @@ -69,6 +89,7 @@ class RdmaBackend { return static_cast(*this).KernelBackend(); } detail::RdmaResources resources_; + LocalSharedPtr engine_; }; } // namespace bycorf #endif // BYCORF_ENABLE_RDMA diff --git a/include/bycorf/io/rdma_reactor.h b/include/bycorf/io/rdma_reactor.h new file mode 100644 index 0000000..867810d --- /dev/null +++ b/include/bycorf/io/rdma_reactor.h @@ -0,0 +1,26 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#if BYCORF_ENABLE_RDMA + +#include "absl/status/statusor.h" +#include "bycorf/base/local_shared_ptr.h" + +namespace bycorf { +class Worker; +namespace rdma { +class RcConnection; +class RcListener; +namespace detail { +class Engine; +// Used only by the private worker facade. Shutdown has two phases because +// io_uring drops outstanding tags without callbacks when the ring is destroyed. +absl::StatusOr> CreateEngine(Worker& worker); +void BeginShutdown(const LocalSharedPtr& engine); +void AfterRingShutdown(const LocalSharedPtr& engine); +} // namespace detail +} // namespace rdma +} // namespace bycorf +#endif diff --git a/include/bycorf/rdma/rc.h b/include/bycorf/rdma/rc.h new file mode 100644 index 0000000..eac7d59 --- /dev/null +++ b/include/bycorf/rdma/rc.h @@ -0,0 +1,113 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#if BYCORF_ENABLE_RDMA +#include + +#include +#include +#include +#include +#include + +#include "bycorf/base/local_shared_ptr.h" +#include "bycorf/io/rdma_memory.h" +#include "bycorf/runtime/task.h" + +namespace bycorf { +class Worker; +namespace rdma { +namespace detail { +struct Connection; +struct Listener; +class Engine; +class RcTestPeer; +} // namespace detail + +// Fixed per-QP capacities; a shared CQ reserves the provider's actual send+recv +// capacities before admitting the QP. No CQ growth or receive-buffer +// allocation. +struct RcOptions { + std::uint32_t send_wr = 128; + std::uint32_t recv_wr = 128; + std::uint32_t cq_entries = 4096; + std::chrono::milliseconds connect_timeout{5000}; +}; + +// Owner-worker-only RC connection. Move-only; destruction requests asynchronous +// close. Await Close() to join resource retirement while the worker is running. +// Worker shutdown also closes all connections before reclaiming coroutine +// frames. Handles and returned Tasks use non-atomic ownership: access, moves +// and destruction stay on the creating thread, including after shutdown. +class RcConnection { + public: + RcConnection() = default; + RcConnection(RcConnection&&) noexcept; + RcConnection& operator=(RcConnection&&) noexcept; + RcConnection(const RcConnection&) = delete; + RcConnection& operator=(const RcConnection&) = delete; + ~RcConnection(); + + // Numeric IPv4/IPv6 only. Copies arguments before returning the lazy task. + // The deadline covers address/route resolution and the RC handshake. Private + // data is limited to 56 bytes for portable RC CM requests and responses. + static Task> Connect( + Worker& worker, std::string_view ip, std::uint16_t port, + RcOptions options = {}, std::span private_data = {}); + // Idempotent; one Close waiter at a time. Completion joins this connection's + // timer/event references and QP/CM ID destruction, not merely cancellation. + // Shared CM/device fd polls belong to the worker, not individual connections. + Task Close(); + bool IsOpen() const noexcept; + // Preserves the specific CM/provider failure; local close reports Cancelled. + absl::Status status() const; + std::span peer_private_data() const noexcept; + // Retain this handle through registered memory use; it is the QP's actual PD. + RdmaProtectionDomain protection_domain() const; + + private: + friend class RcListener; + friend class detail::RcTestPeer; + explicit RcConnection(LocalSharedPtr state); + static Task> ConnectImpl( + LocalSharedPtr engine, sockaddr_storage address, + RcOptions options, std::vector data); + LocalSharedPtr state_; +}; + +// A listener and every accepted connection stay on the listening worker. +// Accepted connections already returned to the application outlive the +// listener. Handles and returned Tasks follow the same creating-thread lifetime +// contract as RcConnection, including destruction after worker shutdown. +class RcListener { + public: + RcListener() = default; + RcListener(RcListener&&) noexcept; + RcListener& operator=(RcListener&&) noexcept; + RcListener(const RcListener&) = delete; + RcListener& operator=(const RcListener&) = delete; + ~RcListener(); + + // Binds a numeric local address; port zero selects an ephemeral port. Options + // and response private data are copied. Requests beyond backlog are rejected. + static absl::StatusOr Listen( + Worker& worker, std::string_view ip, std::uint16_t port, + RcOptions options = {}, std::span private_data = {}, + int backlog = 128); + // One waiter at a time. Returns after establishment or a specific request's + // failure; queued and handshaking children are closed by listener Close(). + Task> Accept(); + Task Close(); + std::uint16_t port() const noexcept; + + private: + friend class detail::RcTestPeer; + static Task> AcceptImpl( + LocalSharedPtr state); + LocalSharedPtr state_; +}; +} // namespace rdma +} // namespace bycorf +#endif diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index 3fe2512..1ca3514 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -91,6 +91,7 @@ class Worker : private IoUringBackend, using Rdma::CancelRdmaEventPoll; using Rdma::OpenRdmaDevice; using Rdma::RegisterRdmaMemory; + using Rdma::SubmitRdmaDeadline; using Rdma::SubmitRdmaEventPoll; #endif struct WakeStats { @@ -386,6 +387,8 @@ class Worker : private IoUringBackend, friend class StorageBackend; #if BYCORF_ENABLE_RDMA friend class RdmaBackend; + friend class rdma::RcConnection; + friend class rdma::RcListener; #endif // Shared-ring access for the CRTP facades; the base conversion is a fixed // address adjustment rather than a load of a stored pointer. diff --git a/src/io/io_uring_backend.cpp b/src/io/io_uring_backend.cpp index 65ff574..60cd0ef 100644 --- a/src/io/io_uring_backend.cpp +++ b/src/io/io_uring_backend.cpp @@ -374,6 +374,15 @@ absl::Status IoUringBackend::SubmitTimeout(const __kernel_timespec& timeout, }); } +absl::Status IoUringBackend::SubmitDeadline(const __kernel_timespec& deadline, + IoCompletion* tag) { + auto* deadline_ptr = const_cast<__kernel_timespec*>(&deadline); + return SubmitOrQueue([deadline_ptr, tag](io_uring_sqe* sqe) { + io_uring_prep_timeout(sqe, deadline_ptr, 0, IORING_TIMEOUT_ABS); + io_uring_sqe_set_data(sqe, tag); + }); +} + io_uring_sqe* IoUringBackend::AcquireSqe() { if (!initialized_) { return nullptr; diff --git a/src/rdma/rc.cpp b/src/rdma/rc.cpp new file mode 100644 index 0000000..cf7c130 --- /dev/null +++ b/src/rdma/rc.cpp @@ -0,0 +1,204 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "bycorf/rdma/rc.h" + +#include + +#include +#include +#include +#include + +#include "bycorf/runtime/worker.h" +#include "rc_internal.h" + +namespace bycorf::rdma { +namespace { +absl::Status Validate(const RcOptions& options, + std::span data) { + if (options.send_wr == 0 || options.recv_wr == 0 || options.cq_entries == 0 || + options.cq_entries > + static_cast(std::numeric_limits::max()) || + options.connect_timeout.count() <= 0 || + options.connect_timeout.count() > std::numeric_limits::max() || + data.size() > 56) + return absl::InvalidArgumentError( + "invalid RDMA capacity, timeout or private data length"); + return absl::OkStatus(); +} +absl::StatusOr Address(std::string_view ip, + std::uint16_t port) { + sockaddr_storage address{}; + std::string text(ip); + auto* v4 = reinterpret_cast(&address); + if (inet_pton(AF_INET, text.c_str(), &v4->sin_addr) == 1) { + v4->sin_family = AF_INET; + v4->sin_port = htons(port); + return address; + } + auto* v6 = reinterpret_cast(&address); + if (inet_pton(AF_INET6, text.c_str(), &v6->sin6_addr) == 1) { + v6->sin6_family = AF_INET6; + v6->sin6_port = htons(port); + return address; + } + return absl::InvalidArgumentError( + "RDMA requires a numeric IPv4 or IPv6 address"); +} +Task> FailedConnect(absl::Status error) { + co_return error; +} + +template +void Drop(const LocalSharedPtr& state) { + if (!state || state->closed) return; + if (auto engine = state->engine.lock()) { + // Destruction can request asynchronous close but cannot migrate ownership. + // After worker shutdown all public states are already retired. + if (!engine->OnOwner() && !engine->stopping) std::terminate(); + engine->Close(state, absl::CancelledError("RDMA handle closed")); + } +} +template +Task CloseImpl(LocalSharedPtr state) { + if (!state || state->closed) co_return absl::OkStatus(); + auto engine = state->engine.lock(); + if (!engine) co_return absl::CancelledError("RDMA owner is gone"); + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError("RDMA close on a different worker"); + engine->Close(state, absl::CancelledError("RDMA handle closed")); + co_return co_await detail::Wait{state->retired}; +} +struct ConnectGuard { + LocalSharedPtr state; + bool delivered = false; + ~ConnectGuard() { + if (!delivered) Drop(state); + } +}; +} // namespace + +RcConnection::RcConnection(LocalSharedPtr state) + : state_(std::move(state)) {} +RcConnection::RcConnection(RcConnection&&) noexcept = default; +RcConnection& RcConnection::operator=(RcConnection&& other) noexcept { + if (this != &other) { + Drop(state_); + state_ = std::move(other.state_); + } + return *this; +} +RcConnection::~RcConnection() { Drop(state_); } +Task> RcConnection::Connect( + Worker& worker, std::string_view ip, std::uint16_t port, RcOptions options, + std::span private_data) { + auto status = Validate(options, private_data); + if (!status.ok()) return FailedConnect(status); + if (port == 0) + return FailedConnect( + absl::InvalidArgumentError("RDMA connect port is zero")); + auto address = Address(ip, port); + if (!address.ok()) return FailedConnect(address.status()); + auto engine = worker.GetRdmaEngine(); + if (!engine.ok()) return FailedConnect(engine.status()); + return ConnectImpl(*engine, *address, options, + {private_data.begin(), private_data.end()}); +} +Task> RcConnection::ConnectImpl( + LocalSharedPtr engine, sockaddr_storage address, + RcOptions options, std::vector data) { + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError( + "RDMA connect on a different worker"); + auto started = engine->Connect(address, options, std::move(data)); + if (!started.ok()) co_return started.status(); + auto c = *started; + ConnectGuard guard{c}; + auto status = co_await detail::Wait{c->connected}; + if (!status.ok()) co_return status; + if (!c->error.ok()) co_return c->error; + guard.delivered = true; + co_return RcConnection(std::move(c)); +} +Task RcConnection::Close() { return CloseImpl(state_); } +bool RcConnection::IsOpen() const noexcept { + return state_ && state_->established && !state_->closing; +} +absl::Status RcConnection::status() const { + return state_ ? state_->error + : absl::FailedPreconditionError("empty RDMA connection"); +} +std::span RcConnection::peer_private_data() const noexcept { + return state_ ? std::span(state_->peer_data) + : std::span{}; +} +RdmaProtectionDomain RcConnection::protection_domain() const { + return state_ && state_->device ? state_->device->domain + : RdmaProtectionDomain{}; +} + +RcListener::RcListener(RcListener&&) noexcept = default; +RcListener& RcListener::operator=(RcListener&& other) noexcept { + if (this != &other) { + Drop(state_); + state_ = std::move(other.state_); + } + return *this; +} +RcListener::~RcListener() { Drop(state_); } +absl::StatusOr RcListener::Listen( + Worker& worker, std::string_view ip, std::uint16_t port, RcOptions options, + std::span private_data, int backlog) { + auto status = Validate(options, private_data); + if (!status.ok()) return status; + if (backlog <= 0) + return absl::InvalidArgumentError("RDMA backlog must be positive"); + auto address = Address(ip, port); + if (!address.ok()) return address.status(); + auto engine = worker.GetRdmaEngine(); + if (!engine.ok()) return engine.status(); + auto started = (*engine)->Listen( + *address, options, {private_data.begin(), private_data.end()}, backlog); + if (!started.ok()) return started.status(); + RcListener listener; + listener.state_ = *started; + return listener; +} +Task> RcListener::Accept() { + return AcceptImpl(state_); +} +Task> RcListener::AcceptImpl( + LocalSharedPtr state) { + if (!state) co_return absl::FailedPreconditionError("empty RDMA listener"); + auto engine = state->engine.lock(); + if (!engine || state->closing) + co_return state->error.ok() ? absl::CancelledError("RDMA listener closed") + : state->error; + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError( + "RDMA accept on a different worker"); + if (state->accepted.waiter) + co_return absl::FailedPreconditionError("concurrent RDMA accept"); + while (state->ready.empty() && !state->closing) { + state->accepted.ready = false; + auto status = co_await detail::Wait{state->accepted}; + if (!status.ok()) co_return status; + } + if (state->closing) co_return state->error; + auto child = state->children.find(state->ready.front()); + assert(child != state->children.end()); + // A failed connection may already be absent from the engine registry. Take + // the listener's ownership before removing either index, without suspending. + auto c = std::move(child->second); + state->ready.pop_front(); + state->children.erase(child); + c->listener.reset(); + if (!c->error.ok()) co_return c->error; + co_return RcConnection(std::move(c)); +} +Task RcListener::Close() { return CloseImpl(state_); } +std::uint16_t RcListener::port() const noexcept { + return state_ ? state_->port : 0; +} +} // namespace bycorf::rdma diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp new file mode 100644 index 0000000..ace942f --- /dev/null +++ b/src/rdma/rc_engine.cpp @@ -0,0 +1,769 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "bycorf/io/rdma_reactor.h" +#include "bycorf/runtime/worker.h" +#include "rc_internal.h" +#include "spdlog/spdlog.h" + +namespace bycorf::rdma::detail { +namespace { +absl::Status Nonblocking(int fd) { + int flags = fcntl(fd, F_GETFL); + if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) + return ErrnoStatus("RDMA channel nonblocking setup"); + return absl::OkStatus(); +} +bool Again() { return errno == EAGAIN || errno == EWOULDBLOCK; } + +template +auto Snapshot(const Map& map) { + std::vector result; + result.reserve(map.size()); + for (const auto& [key, value] : map) result.push_back(value); + return result; +} + +absl::Status CmError(rdma_cm_event_type event, int status) { + // Rejection status can be a transport reason, not errno. Preserve both the + // event and raw status rather than accidentally interpreting it as errno. + return absl::UnavailableError( + absl::StrCat(rdma_event_str(event), " (status ", status, ")")); +} +rdma_conn_param Parameters(const Connection& c) { + rdma_conn_param p{}; + p.private_data = c.local_data.empty() ? nullptr : c.local_data.data(); + p.private_data_len = static_cast(c.local_data.size()); + p.responder_resources = c.responder_resources; + p.initiator_depth = c.initiator_depth; + p.retry_count = 7; + p.rnr_retry_count = 7; + p.qp_num = c.qp->qp_num; + return p; +} +} // namespace + +absl::Status ErrnoStatus(const char* operation) { + int error = errno; + return absl::ErrnoToStatus(error ? error : EIO, + absl::StrCat(operation, " (errno ", error, ")")); +} +void CheckDestroy(int result, const char* operation) { + if (result != 0) { + spdlog::critical("{} failed while retiring RDMA resources: {}", operation, + result); + std::terminate(); + } +} +void Signal::Publish(Worker* worker) { + if (ready) return; + ready = true; + if (worker && waiter) worker->Enqueue(waiter); +} +Wait::~Wait() { + if (signal.waiter == handle) signal.waiter = {}; +} +bool Wait::await_suspend(std::coroutine_handle<> awaiting) { + if (signal.waiter) { + conflict = true; + return false; + } + handle = awaiting; + signal.waiter = awaiting; + return true; +} +absl::Status Wait::await_resume() { + return conflict ? absl::FailedPreconditionError("concurrent RDMA wait") + : absl::OkStatus(); +} + +absl::Status Poll::Arm() { + if (armed || stopping) return absl::OkStatus(); + auto status = engine->worker->SubmitRdmaEventPoll(fd, POLLIN, this); + if (status.ok()) armed = true; + return status; +} +void Poll::Stop() { + if (stopping) return; + stopping = true; + if (armed) { + // Cancellation admission is not retirement. Keep the tag, descriptor and + // resource owner until Complete, or the explicit post-ring-shutdown phase. + const auto status = engine->worker->CancelRdmaEventPoll(this); + if (!status.ok()) engine->worker->RequestStop(); + } +} +void Poll::Complete(Worker&, int result, unsigned) { + // Failure fanout may remove the device cache entry. This event reference + // keeps the containing poll alive until the callback has fully returned. + auto retained = device ? device->self.lock() : nullptr; + armed = false; + if (!stopping) engine->OnPoll(*this, result); + // A cancellation CQE is the retirement boundary; cancellation admission + // alone never permits release of the descriptor or containing Device. + if (device) engine->RetireFailedDevice(*device); +} +void Deadline::Complete(Worker&, int result, unsigned) { + auto it = engine->connections.find(connection->id); + if (it == engine->connections.end()) std::terminate(); + auto retained = it->second; + armed = false; + if (!retained->handshake_done) { + engine->Close(retained, result == -ETIME + ? absl::DeadlineExceededError( + "RDMA connection deadline exceeded") + : absl::UnknownError(absl::StrCat( + "RDMA deadline completion: ", result))); + } + engine->Progress(retained); +} + +Device::~Device() { + if (cq) CheckDestroy(ibv_destroy_cq(cq), "ibv_destroy_cq"); + if (channel) + CheckDestroy(ibv_destroy_comp_channel(channel), "ibv_destroy_comp_channel"); +} +Engine::~Engine() { + // Normal destruction follows AfterRingShutdown. Failed Init has no accepted + // poll, and can release its channel directly. + if (cm) rdma_destroy_event_channel(cm); +} +bool Engine::OnOwner() const { return worker && ThisWorker().self_ == worker; } +absl::Status Engine::Init() { + cm = rdma_create_event_channel(); + if (!cm) return ErrnoStatus("rdma_create_event_channel"); + auto status = Nonblocking(cm->fd); + if (!status.ok()) return status; + cm_poll.engine = this; + cm_poll.fd = cm->fd; + return cm_poll.Arm(); +} +absl::StatusOr> CreateEngine(Worker& worker) { + if (ThisWorker().self_ != &worker) + return absl::FailedPreconditionError("RDMA must run on its owner worker"); + auto engine = MakeLocalShared(worker); + engine->self = engine; + auto status = engine->Init(); + if (!status.ok()) return status; + return engine; +} +void BeginShutdown(const LocalSharedPtr& engine) { + if (engine) engine->BeginShutdown(); +} +void AfterRingShutdown(const LocalSharedPtr& engine) { + if (engine) engine->AfterRingShutdown(); +} + +absl::StatusOr> Engine::Listen( + const sockaddr_storage& address, RcOptions options, + std::vector data, int backlog) { + if (stopping) return absl::CancelledError("RDMA worker is shutting down"); + if (!OnOwner()) + return absl::FailedPreconditionError("RDMA call on a different worker"); + if (!error.ok()) return error; + auto listener = MakeLocalShared(); + listener->engine = self; + listener->options = options; + listener->private_data = std::move(data); + listener->backlog = backlog; + if (rdma_create_id(cm, &listener->id, nullptr, RDMA_PS_TCP)) + return ErrnoStatus("rdma_create_id(listener)"); + auto addr = address; + if (rdma_bind_addr(listener->id, reinterpret_cast(&addr)) || + rdma_listen(listener->id, backlog)) { + auto status = ErrnoStatus("RDMA bind/listen"); + CheckDestroy(rdma_destroy_id(listener->id), "rdma_destroy_id(listener)"); + listener->id = nullptr; + return status; + } + listener->port = ntohs(rdma_get_src_port(listener->id)); + listeners.emplace(listener->id, listener); + return listener; +} +absl::Status Engine::StartTimer(const LocalSharedPtr& c) { + c->expires = std::chrono::steady_clock::now() + c->options.connect_timeout; + c->timer.engine = this; + c->timer.connection = c.get(); + const auto ms = c->options.connect_timeout.count(); + timespec now{}; + if (clock_gettime(CLOCK_MONOTONIC, &now)) return ErrnoStatus("clock_gettime"); + const auto nanos = now.tv_nsec + (ms % 1000) * 1000000; + c->timer.expires = {.tv_sec = now.tv_sec + ms / 1000 + nanos / 1000000000, + .tv_nsec = nanos % 1000000000}; + auto status = worker->SubmitRdmaDeadline(c->timer.expires, &c->timer); + if (status.ok()) c->timer.armed = true; + return status; +} +absl::StatusOr> Engine::Connect( + const sockaddr_storage& address, RcOptions options, + std::vector data) { + if (stopping) return absl::CancelledError("RDMA worker is shutting down"); + if (!OnOwner()) + return absl::FailedPreconditionError("RDMA call on a different worker"); + if (!error.ok()) return error; + auto c = MakeLocalShared(); + c->engine = self; + c->options = options; + c->local_data = std::move(data); + if (rdma_create_id(cm, &c->id, nullptr, RDMA_PS_TCP)) + return ErrnoStatus("rdma_create_id(connect)"); + connections.emplace(c->id, c); + auto status = StartTimer(c); + auto addr = address; + if (status.ok() && + rdma_resolve_addr(c->id, nullptr, reinterpret_cast(&addr), + static_cast(options.connect_timeout.count()))) + status = ErrnoStatus("rdma_resolve_addr"); + if (!status.ok()) Close(c, status); + return c; +} + +absl::StatusOr> Engine::GetDevice(Connection& c) { + auto* selected = c.id->verbs; + if (!selected || !selected->device) + return absl::FailedPreconditionError("CM did not select a verbs device"); + const std::string name = ibv_get_device_name(selected->device); + if (auto it = devices.find(name); it != devices.end()) { + if (!it->second->error.ok()) return it->second->error; + if (it->second->requested_cqe != c.options.cq_entries) + return absl::InvalidArgumentError( + "shared RDMA CQ capacity is already fixed"); + return it->second; + } + auto device = MakeLocalShared(); + device->self = device; + auto* opened = ibv_open_device(selected->device); + if (!opened) return ErrnoStatus("ibv_open_device(RC)"); + device->context = + std::shared_ptr(opened, [](ibv_context* context) { + CheckDestroy(ibv_close_device(context), "ibv_close_device(RC)"); + }); + auto domain = worker->AcquireRdmaDomain(device->context); + if (!domain.ok()) return domain.status(); + device->domain = *domain; + ibv_device_attr attrs{}; + int result = ibv_query_device(opened, &attrs); + if (result) return absl::ErrnoToStatus(result, "ibv_query_device"); + if (c.options.cq_entries > static_cast(attrs.max_cqe)) + return absl::ResourceExhaustedError("requested CQ exceeds device max_cqe"); + device->channel = ibv_create_comp_channel(opened); + if (!device->channel) return ErrnoStatus("ibv_create_comp_channel"); + auto status = Nonblocking(device->channel->fd); + if (!status.ok()) return status; + status = Nonblocking(opened->async_fd); + if (!status.ok()) return status; + device->cq = ibv_create_cq(opened, static_cast(c.options.cq_entries), + device.get(), device->channel, 0); + if (!device->cq) return ErrnoStatus("ibv_create_cq"); + device->requested_cqe = c.options.cq_entries; + device->completion_poll.engine = device->async_poll.engine = this; + device->completion_poll.device = device->async_poll.device = device.get(); + device->completion_poll.kind = Poll::Kind::kCq; + device->completion_poll.fd = device->channel->fd; + device->async_poll.kind = Poll::Kind::kAsync; + device->async_poll.fd = opened->async_fd; + // Retain the device BEFORE admitting any fd poll. A later failure must not + // destroy a tag or fd that the ring may still reference. + devices.emplace(name, device); + result = ibv_req_notify_cq(device->cq, 0); + if (result) status = absl::ErrnoToStatus(result, "ibv_req_notify_cq"); + if (status.ok()) DrainCq(*device); // arm then recheck, including startup + if (status.ok()) status = device->error; + if (status.ok()) status = device->completion_poll.Arm(); + if (status.ok()) status = device->async_poll.Arm(); + if (!status.ok()) { + FailDevice(*device, status); + RetireFailedDevice(*device); + return status; + } + return device; +} +absl::Status Engine::Transition(Connection& c, ibv_qp_state state) { + ibv_qp_attr attr{}; + attr.qp_state = state; + int mask = 0; + if (rdma_init_qp_attr(c.id, &attr, &mask)) + return ErrnoStatus("rdma_init_qp_attr"); + if (state == IBV_QPS_RTR) attr.max_dest_rd_atomic = c.responder_resources; + if (state == IBV_QPS_RTS) attr.max_rd_atomic = c.initiator_depth; + const int result = ibv_modify_qp(c.qp, &attr, mask); + return result ? absl::ErrnoToStatus(result, "ibv_modify_qp(RC)") + : absl::OkStatus(); +} +absl::Status Engine::CreateQp(const LocalSharedPtr& c) { + auto found = GetDevice(*c); + if (!found.ok()) return found.status(); + auto device = *found; + const std::uint64_t requested = + std::uint64_t(c->options.send_wr) + c->options.recv_wr; + if (requested > + static_cast(device->cq->cqe) - device->reserved) + return absl::ResourceExhaustedError("shared RDMA CQ budget exhausted"); + ibv_qp_init_attr attr{}; + attr.qp_type = IBV_QPT_RC; + attr.send_cq = attr.recv_cq = device->cq; + attr.cap.max_send_wr = c->options.send_wr; + attr.cap.max_recv_wr = c->options.recv_wr; + attr.cap.max_send_sge = attr.cap.max_recv_sge = 1; + attr.sq_sig_all = 1; + c->qp = ibv_create_qp(device->domain.native_pd(), &attr); + if (!c->qp) return ErrnoStatus("ibv_create_qp(RC)"); + // Providers may round capacities up. Until the actual budget is reserved, + // this QP remains unconnected with no work posted, so it cannot fill the CQ. + const std::uint64_t actual = + std::uint64_t(attr.cap.max_send_wr) + attr.cap.max_recv_wr; + if (actual > static_cast(device->cq->cqe) - device->reserved) { + CheckDestroy(ibv_destroy_qp(c->qp), "ibv_destroy_qp(over budget)"); + c->qp = nullptr; + return absl::ResourceExhaustedError( + "provider QP capacities exceed shared CQ budget"); + } + c->device = device; + c->budget = actual; + device->reserved += actual; + // External-QP CM mode: never assign id->qp. CM routes/negotiates, while the + // worker's own context owns QP/PD/MR/CQ and the sole async-event reader. + return Transition(*c, IBV_QPS_INIT); +} + +void Engine::Established(const LocalSharedPtr& c) { + if (std::chrono::steady_clock::now() >= c->expires) { + Close(c, absl::DeadlineExceededError("RDMA connection deadline exceeded")); + return; + } + c->established = true; + c->handshake_done = true; + if (c->timer.armed) { + auto status = worker->CancelRdmaEventPoll(&c->timer); + if (!status.ok()) { + Close(c, status); + return; + } + } + Progress(c); +} +void Engine::HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, + rdma_cm_event_type event, int status, + std::vector data, rdma_conn_param peer) { + if (event == RDMA_CM_EVENT_DEVICE_REMOVAL) { + // Retain endpoints before fanout: failing the device can erase their map + // entries and destroy the CM ID. Nonzero status still means device scope. + auto ci = connections.find(id); + auto li = listeners.find(id); + auto connection = ci == connections.end() ? nullptr : ci->second; + auto listener = li == listeners.end() ? nullptr : li->second; + const auto error = CmError(event, status); + if (id->verbs && id->verbs->device) { + auto found = devices.find(ibv_get_device_name(id->verbs->device)); + if (found != devices.end()) FailDevice(*found->second, error); + } + if (connection) Close(connection, error); + if (listener) Close(listener, error); + return; + } + if (event == RDMA_CM_EVENT_CONNECT_REQUEST) { + auto it = listeners.find(listen_id); + if (stopping || it == listeners.end() || it->second->closing || + it->second->children.size() >= + static_cast(it->second->backlog)) { + (void)rdma_reject(id, nullptr, 0); + CheckDestroy(rdma_destroy_id(id), "rdma_destroy_id(rejected request)"); + return; + } + auto listener = it->second; + auto c = MakeLocalShared(); + c->engine = self; + c->listener = listener; + c->id = id; + c->options = listener->options; + c->local_data = listener->private_data; + c->peer_data = std::move(data); + c->initiator_depth = std::min(1, peer.responder_resources); + c->responder_resources = std::min(1, peer.initiator_depth); + connections.emplace(id, c); + listener->children.emplace(c.get(), c); + auto result = StartTimer(c); + if (result.ok()) result = CreateQp(c); + if (result.ok()) result = Transition(*c, IBV_QPS_RTR); + if (result.ok()) result = Transition(*c, IBV_QPS_RTS); + if (result.ok()) { + auto param = Parameters(*c); + if (rdma_accept(c->id, ¶m)) + result = ErrnoStatus("rdma_accept"); + else + c->cm_started = true; + } + if (!result.ok()) { + (void)rdma_reject(id, nullptr, 0); + Close(c, result); + } + return; + } + auto it = connections.find(id); + if (it == connections.end()) { + if (auto listener = listeners.find(id); listener != listeners.end()) + Close(listener->second, CmError(event, status)); + return; + } + auto c = it->second; + if (c->closing) return; + if (status || event == RDMA_CM_EVENT_REJECTED || + event == RDMA_CM_EVENT_CONNECT_ERROR || + event == RDMA_CM_EVENT_ADDR_ERROR || event == RDMA_CM_EVENT_ROUTE_ERROR || + event == RDMA_CM_EVENT_UNREACHABLE || + event == RDMA_CM_EVENT_DISCONNECTED) { + if (!data.empty()) c->peer_data = std::move(data); + Close(c, CmError(event, status)); + return; + } + absl::Status result; + switch (event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + if (rdma_resolve_route( + id, static_cast(c->options.connect_timeout.count()))) + result = ErrnoStatus("rdma_resolve_route"); + break; + case RDMA_CM_EVENT_ROUTE_RESOLVED: + result = CreateQp(c); + if (result.ok()) { + auto param = Parameters(*c); + if (rdma_connect(id, ¶m)) + result = ErrnoStatus("rdma_connect"); + else + c->cm_started = true; + } + break; + case RDMA_CM_EVENT_CONNECT_RESPONSE: + c->peer_data = std::move(data); + c->initiator_depth = + std::min(c->initiator_depth, peer.responder_resources); + c->responder_resources = + std::min(c->responder_resources, peer.initiator_depth); + result = Transition(*c, IBV_QPS_INIT); + if (result.ok()) result = Transition(*c, IBV_QPS_RTR); + if (result.ok()) result = Transition(*c, IBV_QPS_RTS); + if (result.ok() && rdma_establish(id)) + result = ErrnoStatus("rdma_establish"); + if (result.ok()) Established(c); + break; + case RDMA_CM_EVENT_ESTABLISHED: + Established(c); + break; + case RDMA_CM_EVENT_DEVICE_REMOVAL: + if (c->device) + FailDevice(*c->device, CmError(event, status)); + else + Close(c, CmError(event, status)); + break; + case RDMA_CM_EVENT_TIMEWAIT_EXIT: + break; + default: + result = CmError(event, status); + break; + } + if (!result.ok()) Close(c, result); +} +void Engine::DrainCm() { + for (;;) { + rdma_cm_event* event = nullptr; + if (rdma_get_cm_event(cm, &event)) { + if (errno == EINTR) continue; + if (!Again()) [[unlikely]] { + FailAll(ErrnoStatus("rdma_get_cm_event")); + } + return; + } + auto* id = event->id; + auto* listener = event->listen_id; + const auto kind = event->event; + const int status = event->status; + std::vector data; + rdma_conn_param peer{}; + if (kind == RDMA_CM_EVENT_CONNECT_REQUEST || + kind == RDMA_CM_EVENT_CONNECT_RESPONSE || + kind == RDMA_CM_EVENT_ESTABLISHED || kind == RDMA_CM_EVENT_REJECTED) { + peer = event->param.conn; + peer.private_data = nullptr; + const auto* first = + static_cast(event->param.conn.private_data); + if (first && event->param.conn.private_data_len) + data.assign(first, first + event->param.conn.private_data_len); + } + // Copy before ack, then dispatch. No handler may destroy an ID while an + // unacknowledged CM event still refers to it (rdma_destroy_id would wait). + CheckDestroy(rdma_ack_cm_event(event), "rdma_ack_cm_event"); + HandleCm(id, listener, kind, status, std::move(data), peer); + } +} + +void Engine::DrainCq(Device& device) { + ibv_wc completions[32]; + for (;;) { + const int count = ibv_poll_cq(device.cq, 32, completions); + if (count < 0) [[unlikely]] { + FailDevice(device, absl::UnavailableError("ibv_poll_cq failed")); + return; + } + // Empty is the normal end of a drain, including notification rechecks. + if (!count) return; + for (int i = 0; i < count; ++i) { + const auto& wc = completions[i]; + // No public data-posting API exists yet. Error completions still + // terminate their connection; success entries are consumed so + // notifications rearm. + if (wc.status == IBV_WC_SUCCESS) continue; + for (auto& c : Snapshot(connections)) { + if (c->device.get() == &device && c->qp && c->qp->qp_num == wc.qp_num) { + Close(c, absl::UnavailableError(absl::StrCat( + "RDMA completion: ", ibv_wc_status_str(wc.status), + " (vendor ", wc.vendor_err, ")"))); + } + } + } + } +} +void Engine::DrainNotifications(Device& device, bool rearm) { + for (;;) { + ibv_cq* cq = nullptr; + void* context = nullptr; + if (ibv_get_cq_event(device.channel, &cq, &context)) { + if (errno == EINTR) continue; + if (!Again() && rearm) [[unlikely]] { + FailDevice(device, ErrnoStatus("ibv_get_cq_event")); + } + return; + } + ibv_ack_cq_events(cq, 1); + if (cq != device.cq) [[unlikely]] { + FailDevice(device, absl::InternalError( + "unexpected CQ on worker completion channel")); + continue; + } + if (rearm && device.error.ok()) { + const int result = ibv_req_notify_cq(cq, 0); + if (result) [[unlikely]] { + FailDevice(device, absl::ErrnoToStatus(result, "ibv_req_notify_cq")); + continue; + } + // Notification is one-shot. Rearm BEFORE draining: a completion racing + // this drain is either consumed here or leaves a channel event to wake + // us. + DrainCq(device); + } + } +} +void Engine::DrainAsync(Device& device) { + for (;;) { + ibv_async_event event{}; + if (ibv_get_async_event(device.context.get(), &event)) { + if (errno == EINTR) continue; + if (!Again()) [[unlikely]] { + FailDevice(device, ErrnoStatus("ibv_get_async_event")); + } + return; + } + const auto type = event.event_type; + const auto element = event.element; + ibv_ack_async_event(&event); + auto error = absl::UnavailableError( + absl::StrCat("RDMA device event: ", ibv_event_type_str(type))); + if (type == IBV_EVENT_DEVICE_FATAL || type == IBV_EVENT_CQ_ERR) { + FailDevice(device, error); + } else if (type == IBV_EVENT_QP_FATAL || type == IBV_EVENT_QP_REQ_ERR || + type == IBV_EVENT_QP_ACCESS_ERR || type == IBV_EVENT_PORT_ERR) { + for (auto& c : Snapshot(connections)) { + if (c->device.get() == &device && + ((type == IBV_EVENT_PORT_ERR && + c->id->port_num == element.port_num) || + (type != IBV_EVENT_PORT_ERR && c->qp == element.qp))) + Close(c, error); + } + } + } +} +void Engine::OnPoll(Poll& poll, int result) { + if (result < 0 || (result & (POLLERR | POLLHUP | POLLNVAL))) [[unlikely]] { + auto error = + absl::UnavailableError(absl::StrCat("RDMA fd poll failed: ", result)); + if (poll.device) + FailDevice(*poll.device, error); + else + FailAll(error); + return; + } + switch (poll.kind) { + case Poll::Kind::kCm: + DrainCm(); + break; + case Poll::Kind::kCq: + DrainNotifications(*poll.device, true); + break; + case Poll::Kind::kAsync: + DrainAsync(*poll.device); + break; + } + auto status = poll.Arm(); + if (!status.ok()) [[unlikely]] { + if (poll.device) + FailDevice(*poll.device, status); + else + FailAll(status); + } +} +void Engine::FailDevice(Device& device, const absl::Status& error) { + if (!device.error.ok()) return; + device.error = error; + device.completion_poll.Stop(); + device.async_poll.Stop(); + for (auto& c : Snapshot(connections)) + if (c->device.get() == &device) Close(c, error); +} +void Engine::RetireFailedDevice(Device& device) { + if (stopping || device.error.ok() || device.reserved != 0 || + device.completion_poll.armed || device.async_poll.armed) + return; + DrainNotifications(device, false); + DrainAsync(device); + for (auto it = devices.begin(); it != devices.end(); ++it) { + if (it->second.get() == &device) { + devices.erase(it); + return; + } + } +} +void Engine::FailAll(const absl::Status& error) { + if (this->error.ok()) this->error = error; + cm_poll.Stop(); + for (auto& listener : Snapshot(listeners)) Close(listener, error); + for (auto& c : Snapshot(connections)) Close(c, error); +} + +void Engine::Close(LocalSharedPtr c, absl::Status reason) { + if (c->closing || c->closed) return; + c->closing = true; + c->handshake_done = true; + if (c->error.ok()) c->error = std::move(reason); + if (c->cm_started) (void)rdma_disconnect(c->id); + // External QPs are not transitioned by rdma_disconnect. Posting stops before + // ERR, and CQ capacity is not returned until the QP is destroyed and drained. + if (c->qp) { + ibv_qp_attr attr{}; + attr.qp_state = IBV_QPS_ERR; + (void)ibv_modify_qp(c->qp, &attr, IBV_QP_STATE); + } + if (c->timer.armed) { + auto status = worker->CancelRdmaEventPoll(&c->timer); + if (!status.ok()) worker->RequestStop(); + } + Progress(c); +} +void Engine::Progress(LocalSharedPtr c) { + if (c->timer.armed || c->closed) return; + if (c->closing) { + // Erase before any CQ drain can fail another connection and reenter this + // path. The local shared reference retains the object through callbacks. + connections.erase(c->id); + if (c->qp) { + auto* qp = std::exchange(c->qp, nullptr); + CheckDestroy(ibv_destroy_qp(qp), "ibv_destroy_qp(RC)"); + DrainCq(*c->device); + c->device->reserved -= c->budget; + c->budget = 0; + RetireFailedDevice(*c->device); + } + CheckDestroy(rdma_destroy_id(c->id), "rdma_destroy_id(RC)"); + c->id = nullptr; + c->device.reset(); + c->closed = true; + c->retired.Publish(wake_worker()); + } + if (c->handshake_done) { + c->connected.Publish(wake_worker()); + if (auto listener = c->listener.lock()) { + if (!c->queued && !listener->closing) { + c->queued = true; + listener->ready.push_back(c.get()); + listener->accepted.Publish(wake_worker()); + } + Progress(listener); + } + } +} +void Engine::Close(LocalSharedPtr listener, absl::Status reason) { + if (listener->closing || listener->closed) return; + listener->closing = true; + listener->error = reason; + // No per-listener fd tag exists: the shared CM poll references only Engine. + // DrainCm has acknowledged its event before this ID can be destroyed. + CheckDestroy(rdma_destroy_id(listener->id), "rdma_destroy_id(listener)"); + listener->id = nullptr; + for (auto& c : Snapshot(listener->children)) Close(c, reason); + listener->accepted.Publish(wake_worker()); + Progress(listener); +} +void Engine::Progress(LocalSharedPtr listener) { + if (!listener->closing || listener->closed) return; + for (const auto& [key, c] : listener->children) + if (!c->closed) return; + listener->ready.clear(); + listener->children.clear(); + for (auto it = listeners.begin(); it != listeners.end(); ++it) { + if (it->second == listener) { + listeners.erase(it); + break; + } + } + listener->closed = true; + listener->retired.Publish(wake_worker()); +} +void Engine::BeginShutdown() { + if (stopping) return; + stopping = true; + cm_poll.Stop(); + for (auto& [name, device] : devices) { + device->completion_poll.Stop(); + device->async_poll.Stop(); + } + const auto error = absl::CancelledError("RDMA worker shutdown"); + for (auto& listener : Snapshot(listeners)) Close(listener, error); + for (auto& c : Snapshot(connections)) Close(c, error); +} +void Engine::AfterRingShutdown() { + // io_uring_queue_exit has joined kernel access, including queued/unsubmitted + // tags which never got callbacks. Only now may retained fd owners disappear. + cm_poll.armed = false; + for (auto& [name, device] : devices) { + device->completion_poll.armed = device->async_poll.armed = false; + DrainAsync(*device); + } + if (cm) DrainCm(); + for (auto& c : Snapshot(connections)) { + c->timer.armed = false; + Progress(c); + } + for (auto& listener : Snapshot(listeners)) Progress(listener); + for (auto& [name, device] : devices) { + DrainNotifications(*device, false); + DrainAsync(*device); + // Public connection states can survive shutdown; release device references + // there too, but exported PD/MR handles independently retain their context. + } + devices.clear(); + if (cm) { + rdma_destroy_event_channel(cm); + cm = nullptr; + } + worker = nullptr; +} +} // namespace bycorf::rdma::detail diff --git a/src/rdma/rc_internal.h b/src/rdma/rc_internal.h new file mode 100644 index 0000000..4736b65 --- /dev/null +++ b/src/rdma/rc_internal.h @@ -0,0 +1,165 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "bycorf/base/inline_ring_queue.h" +#include "bycorf/io/completion.h" +#include "bycorf/rdma/rc.h" + +namespace bycorf::rdma::detail { +struct Device; +struct Signal { + bool ready = false; + std::coroutine_handle<> waiter{}; + void Publish(Worker* worker); +}; + +// The waiter is owned by its coroutine, not by an fd tag. Shutdown may destroy +// that coroutine only after the ring is quiesced; this guard detaches it. +struct Wait { + Signal& signal; + std::coroutine_handle<> handle{}; + bool conflict = false; + ~Wait(); + bool await_ready() const noexcept { return signal.ready; } + bool await_suspend(std::coroutine_handle<> awaiting); + absl::Status await_resume(); +}; + +struct Poll final : IoCompletion { + enum class Kind { kCm, kCq, kAsync }; + Engine* engine = nullptr; + Device* device = nullptr; + Kind kind = Kind::kCm; + int fd = -1; + bool armed = false; + bool stopping = false; + absl::Status Arm(); + void Stop(); + void Complete(Worker&, int result, unsigned flags) override; +}; + +struct Deadline final : IoCompletion { + Engine* engine = nullptr; + Connection* connection = nullptr; + bool armed = false; + __kernel_timespec expires{}; + void Complete(Worker&, int result, unsigned flags) override; +}; + +struct Connection { + LocalWeakPtr engine; + LocalWeakPtr listener; + rdma_cm_id* id = nullptr; + ibv_qp* qp = nullptr; + LocalSharedPtr device; + std::uint64_t budget = 0; + RcOptions options; + std::vector local_data, peer_data; + Deadline timer; + std::chrono::steady_clock::time_point expires; + bool established = false; + bool cm_started = false; + std::uint8_t initiator_depth = 1; + std::uint8_t responder_resources = 1; + bool handshake_done = false; + bool closing = false; + bool queued = false; + bool closed = false; + absl::Status error; + Signal connected, retired; +}; + +struct Listener { + LocalWeakPtr engine; + rdma_cm_id* id = nullptr; + RcOptions options; + std::vector private_data; + std::uint16_t port = 0; + int backlog = 0; + bool closing = false; + bool closed = false; + absl::Status error; + // Own unaccepted children even after failure removes them from the engine. + // ready only borrows these states: Accept transfers ownership before erasing + // a child, and listener retirement clears ready before releasing children. + absl::flat_hash_map> children; + InlineRingQueue ready; + Signal accepted, retired; +}; + +struct Device { + LocalWeakPtr self; + std::shared_ptr context; + RdmaProtectionDomain domain; + ibv_comp_channel* channel = nullptr; + ibv_cq* cq = nullptr; + std::uint32_t requested_cqe = 0; + std::uint64_t reserved = 0; + absl::Status error; + Poll completion_poll, async_poll; + // CM has process-shared contexts. This context is opened privately on the + // same actual device so its async event stream has exactly one worker reader. + ~Device(); +}; + +class Engine { + public: + LocalWeakPtr self; + explicit Engine(Worker& owner) : worker(&owner) {} + ~Engine(); + absl::Status Init(); + absl::StatusOr> Listen( + const sockaddr_storage& address, RcOptions options, + std::vector data, int backlog); + absl::StatusOr> Connect( + const sockaddr_storage& address, RcOptions options, + std::vector data); + void Close(LocalSharedPtr connection, absl::Status reason); + void Close(LocalSharedPtr listener, absl::Status reason); + void Progress(LocalSharedPtr connection); + void Progress(LocalSharedPtr listener); + void OnPoll(Poll& poll, int result); + void DrainCm(); + void DrainCq(Device& device); + void DrainAsync(Device& device); + void FailDevice(Device& device, const absl::Status& error); + void RetireFailedDevice(Device& device); + void BeginShutdown(); + void AfterRingShutdown(); + bool OnOwner() const; + Worker* wake_worker() const { return stopping ? nullptr : worker; } + + Worker* worker; + bool stopping = false; + absl::Status error; + rdma_event_channel* cm = nullptr; + Poll cm_poll; + absl::flat_hash_map> connections; + absl::flat_hash_map> listeners; + absl::flat_hash_map> devices; + + private: + absl::Status StartTimer(const LocalSharedPtr& connection); + absl::Status CreateQp(const LocalSharedPtr& connection); + absl::Status Transition(Connection& connection, ibv_qp_state state); + absl::StatusOr> GetDevice(Connection& connection); + void HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, rdma_cm_event_type event, + int status, std::vector data, rdma_conn_param peer); + void Established(const LocalSharedPtr& connection); + void FailAll(const absl::Status& error); + void DrainNotifications(Device& device, bool rearm); +}; + +absl::Status ErrnoStatus(const char* operation); +void CheckDestroy(int result, const char* operation); +} // namespace bycorf::rdma::detail diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index 5a82d84..76cb8a1 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -237,17 +237,19 @@ absl::Status Worker::Init(const WorkerOptions& options) { void Worker::Shutdown() { #if BYCORF_ENABLE_RDMA - // Registration can be used before Init; its handles retain verbs resources - // independently of the ring. Never invalidate a live MR during shutdown. + // Stop RDMA admission first but retain fd tags and device resources until + // kernel shutdown confirms that no poll or timeout can reference them. Rdma::Shutdown(); #endif - if (!initialized_) { - return; + if (initialized_) { + Storage::Shutdown(); + Network::Shutdown(); + KernelBackend().Shutdown(); + initialized_ = false; } - Storage::Shutdown(); - Network::Shutdown(); - KernelBackend().Shutdown(); - initialized_ = false; +#if BYCORF_ENABLE_RDMA + Rdma::AfterRingShutdown(); +#endif } void Worker::Spawn(Task task) { diff --git a/tests/io_uring_backpressure_test.cpp b/tests/io_uring_backpressure_test.cpp index eacf08a..0955d30 100644 --- a/tests/io_uring_backpressure_test.cpp +++ b/tests/io_uring_backpressure_test.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -68,6 +70,25 @@ void Drain(Backend& backend, Predicate done) { } } +TEST(IoUringBackpressure, AbsoluteDeadlinesSurviveDeferredSubmission) { + // These objects outlive the backend, including on an assertion failure. + std::array tags; + timespec now{}; + ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &now), 0); + const __kernel_timespec expired{.tv_sec = now.tv_sec, .tv_nsec = now.tv_nsec}; + Fixture f; + ASSERT_NO_FATAL_FAILURE(f.Init()); + for (auto& tag : tags) + ASSERT_TRUE(f.backend.SubmitDeadline(expired, &tag).ok()); + // Sixteen requests exceed this fixture's two-entry SQ. Every request must + // expire after submission rather than restarting a relative timeout. + Drain(f.backend, [&] { + return std::all_of(tags.begin(), tags.end(), + [](const auto& tag) { return tag.calls == 1; }); + }); + for (const auto& tag : tags) EXPECT_EQ(tag.result, -ETIME); +} + TEST(IoUringBackpressure, SendsAndWake) { Fixture source, peer; ASSERT_NO_FATAL_FAILURE(source.Init()); diff --git a/tests/local_shared_ptr_test.cpp b/tests/local_shared_ptr_test.cpp new file mode 100644 index 0000000..cd65eb9 --- /dev/null +++ b/tests/local_shared_ptr_test.cpp @@ -0,0 +1,148 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "bycorf/base/local_shared_ptr.h" + +#include + +#include +#include + +namespace bycorf { +namespace { +struct Tracked { + explicit Tracked(int& destroyed) : destroyed(destroyed) {} + ~Tracked() { ++destroyed; } + int& destroyed; +}; + +TEST(LocalSharedPtr, StrongCopiesMovesAndConstConversion) { + int destroyed = 0; + auto first = MakeLocalShared(destroyed); + auto second = first; + LocalSharedPtr view = first; + EXPECT_EQ(first.use_count(), 3U); + auto moved = std::move(second); + EXPECT_FALSE(second); + EXPECT_EQ(moved.get(), first.get()); + EXPECT_EQ(first.use_count(), 3U); + first.reset(); + moved = nullptr; + EXPECT_EQ(destroyed, 0); + EXPECT_EQ(view.use_count(), 1U); + view.reset(); + EXPECT_EQ(destroyed, 1); +} + +TEST(LocalSharedPtr, WeakLockDoesNotResurrectDestroyedObject) { + int destroyed = 0; + auto first = MakeLocalShared(destroyed); + LocalWeakPtr weak = first; + auto copy = weak; + auto moved = std::move(copy); + EXPECT_TRUE(copy.expired()); + auto retained = moved.lock(); + first.reset(); + EXPECT_EQ(destroyed, 0); + EXPECT_FALSE(weak.expired()); + retained.reset(); + EXPECT_EQ(destroyed, 1); + EXPECT_TRUE(weak.expired()); + EXPECT_FALSE(moved.lock()); + weak.reset(); + moved.reset(); + EXPECT_EQ(destroyed, 1); +} + +struct SelfObserved { + LocalWeakPtr self; + bool& destroyed; + explicit SelfObserved(bool& destroyed) : destroyed(destroyed) {} + ~SelfObserved() { + EXPECT_TRUE(self.expired()); + EXPECT_FALSE(self.lock()); + // The implicit weak reference must protect the control while destruction + // drops the last explicit weak reference. + self.reset(); + destroyed = true; + } +}; +TEST(LocalSharedPtr, SelfWeakReferenceCanReleaseDuringDestruction) { + bool destroyed = false; + auto state = MakeLocalShared(destroyed); + state->self = state; + state.reset(); + EXPECT_TRUE(destroyed); +} + +struct Incomplete; +struct OpaqueHandle { + LocalSharedPtr strong; + LocalWeakPtr weak; + ~OpaqueHandle() = default; +}; +// Instantiate destruction while the pointee is still incomplete, as in the +// private worker facade and public RC handles. +void DestroyOpaque(OpaqueHandle* handle) { delete handle; } +struct Incomplete { + int& destroyed; + explicit Incomplete(int& destroyed) : destroyed(destroyed) {} + ~Incomplete() { ++destroyed; } +}; +TEST(LocalSharedPtr, IncompletePointeeDestruction) { + int destroyed = 0; + auto* handle = new OpaqueHandle; + handle->strong = MakeLocalShared(destroyed); + handle->weak = handle->strong; + DestroyOpaque(handle); + EXPECT_EQ(destroyed, 1); +} + +struct Throwing { + Tracked member; + explicit Throwing(int& destroyed) : member(destroyed) { + throw std::runtime_error("constructor failed"); + } +}; +TEST(LocalSharedPtr, ConstructionFailureUnwindsMembers) { + int destroyed = 0; + EXPECT_THROW(MakeLocalShared(destroyed), std::runtime_error); + EXPECT_EQ(destroyed, 1); +} + +#ifndef NDEBUG +TEST(LocalSharedPtrDeathTest, ForeignThreadCopyAndReleaseAreRejected) { + auto state = MakeLocalShared(7); + EXPECT_DEATH( + { + std::thread other([&] { auto copy = state; }); + other.join(); + }, + "outside its owner thread"); + EXPECT_DEATH( + { + std::thread other([&] { state.reset(); }); + other.join(); + }, + "outside its owner thread"); +} +TEST(LocalSharedPtrDeathTest, ForeignThreadWeakLockAndReleaseAreRejected) { + auto state = MakeLocalShared(7); + LocalWeakPtr weak = state; + EXPECT_DEATH( + { + std::thread other([&] { auto retained = weak.lock(); }); + other.join(); + }, + "outside its owner thread"); + state.reset(); + EXPECT_DEATH( + { + std::thread other([&] { weak.reset(); }); + other.join(); + }, + "outside its owner thread"); +} +#endif +} // namespace +} // namespace bycorf diff --git a/tests/rdma_fake.cpp b/tests/rdma_fake.cpp new file mode 100644 index 0000000..1d6c801 --- /dev/null +++ b/tests/rdma_fake.cpp @@ -0,0 +1,403 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "rdma_fake.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct CmEvent { + rdma_cm_event event{}; + std::vector data; +}; +struct Id { + bool listening = false; + rdma_cm_id* peer = nullptr; +}; +struct Cq { + bool armed = false; + unsigned inject_qp = 0; + int polls = 0, notifications = 0; + std::deque completions; +}; +struct Fake { + std::unordered_map>> + channels; + std::unordered_map> delivered; + std::unordered_map ids; + std::unordered_map> contexts; + std::unordered_map> + completion_channels; + std::unordered_map cqs; + std::unordered_set pds; + std::unordered_set qps; + bool suppress_establish = false, suppress_connect = false, fail_qp = false, + fail_notify = false; + unsigned qp_extra = 0, next_qp = 1; + std::uint16_t next_port = 10000; + int cq_acks = 0, async_acks = 0; +} f; +ibv_device device{}; +ibv_context cm_context{}; + +int EventFd() { return eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC | EFD_SEMAPHORE); } +void Notify(int fd) { + std::uint64_t one = 1; + EXPECT_EQ(write(fd, &one, sizeof(one)), sizeof(one)); +} +void Consume(int fd) { + std::uint64_t one; + EXPECT_EQ(read(fd, &one, sizeof(one)), sizeof(one)); +} +int Empty() { + errno = EAGAIN; + return -1; +} +void Event(rdma_cm_id* id, rdma_cm_event_type type, + rdma_cm_id* listener = nullptr, const void* data = nullptr, + std::size_t length = 0, int status = 0) { + auto e = std::make_unique(); + e->event.id = id; + e->event.listen_id = listener; + e->event.event = type; + e->event.status = status; + if (data && length) { + auto* first = static_cast(data); + e->data.assign(first, first + length); + e->event.param.conn.private_data = e->data.data(); + e->event.param.conn.private_data_len = static_cast(length); + } + e->event.param.conn.initiator_depth = + e->event.param.conn.responder_resources = 1; + f.channels.at(id->channel).push_back(std::move(e)); + Notify(id->channel->fd); +} +int PollCq(ibv_cq* cq, int capacity, ibv_wc* output) { + auto& state = f.cqs.at(cq); + ++state.polls; + int count = 0; + while (count < capacity && !state.completions.empty()) { + output[count++] = state.completions.front(); + state.completions.pop_front(); + } + return count; +} +int ArmCq(ibv_cq* cq, int solicited) { + EXPECT_EQ(solicited, 0); + if (std::exchange(f.fail_notify, false)) return EIO; + auto& state = f.cqs.at(cq); + state.armed = true; + if (auto qp = std::exchange(state.inject_qp, 0U)) + rdma_fake::Completion(cq, qp); + return 0; +} +} // namespace + +namespace rdma_fake { +void Reset() { + f = {}; + cm_context.device = &device; +} +void CheckEmpty() { + EXPECT_TRUE(f.channels.empty()); + EXPECT_TRUE(f.delivered.empty()); + EXPECT_TRUE(f.ids.empty()); + EXPECT_TRUE(f.contexts.empty()); + EXPECT_TRUE(f.completion_channels.empty()); + EXPECT_TRUE(f.cqs.empty()); + EXPECT_TRUE(f.pds.empty()); + EXPECT_TRUE(f.qps.empty()); +} +void Cm(rdma_cm_id* id, rdma_cm_event_type event, int status) { + Event(id, event, nullptr, nullptr, 0, status); +} +void SuppressEstablish(bool value) { f.suppress_establish = value; } +void SuppressConnect(bool value) { f.suppress_connect = value; } +void RoundQp(unsigned extra) { f.qp_extra = extra; } +void FailNextQp() { f.fail_qp = true; } +void FailNextNotify() { f.fail_notify = true; } +void Completion(ibv_cq* cq, std::uint32_t qp, ibv_wc_status status) { + auto& state = f.cqs.at(cq); + ibv_wc wc{}; + wc.qp_num = qp; + wc.status = status; + state.completions.push_back(wc); + if (std::exchange(state.armed, false)) { + f.completion_channels.at(cq->channel).push_back(cq); + ++state.notifications; + Notify(cq->channel->fd); + } +} +void Async(ibv_context* context, ibv_event_type type, ibv_qp* qp) { + ibv_async_event e{}; + e.event_type = type; + if (type == IBV_EVENT_CQ_ERR) + e.element.cq = qp->send_cq; + else if (type != IBV_EVENT_DEVICE_FATAL) + e.element.qp = qp; + f.contexts.at(context).push_back(e); + Notify(context->async_fd); +} +void CompleteOnArm(ibv_cq* cq, std::uint32_t qp) { + f.cqs.at(cq).inject_qp = qp; +} +int CqPolls(ibv_cq* cq) { return f.cqs.at(cq).polls; } +int Notifications(ibv_cq* cq) { return f.cqs.at(cq).notifications; } +int AckedCqEvents() { return f.cq_acks; } +int AckedAsyncEvents() { return f.async_acks; } +int LiveQps() { return static_cast(f.qps.size()); } +int LiveIds() { return static_cast(f.ids.size()); } +} // namespace rdma_fake + +extern "C" { +rdma_event_channel* __wrap_rdma_create_event_channel() { + auto* c = new rdma_event_channel{}; + c->fd = EventFd(); + f.channels.emplace(c, std::deque>{}); + return c; +} +void __wrap_rdma_destroy_event_channel(rdma_event_channel* c) { + EXPECT_TRUE(f.channels.at(c).empty()); + f.channels.erase(c); + close(c->fd); + delete c; +} +int __wrap_rdma_create_id(rdma_event_channel* channel, rdma_cm_id** output, + void* context, rdma_port_space ps) { + EXPECT_EQ(ps, RDMA_PS_TCP); + auto* id = new rdma_cm_id{}; + id->channel = channel; + id->context = context; + id->verbs = &cm_context; + id->port_num = 1; + id->qp_type = IBV_QPT_RC; + f.ids.emplace(id, Id{}); + *output = id; + return 0; +} +int __wrap_rdma_destroy_id(rdma_cm_id* id) { + for (const auto& [key, event] : f.delivered) { + EXPECT_NE(event->event.id, id); + EXPECT_NE(event->event.listen_id, id); + } + auto& queue = f.channels.at(id->channel); + for (auto it = queue.begin(); it != queue.end();) { + if ((*it)->event.id == id) { + Consume(id->channel->fd); + it = queue.erase(it); + } else + ++it; + } + for (auto& [key, value] : f.ids) + if (value.peer == id) value.peer = nullptr; + EXPECT_EQ(f.ids.erase(id), 1U); + delete id; + return 0; +} +int __wrap_rdma_bind_addr(rdma_cm_id* id, sockaddr* address) { + auto* local = reinterpret_cast(&id->route.addr.src_addr); + *local = *reinterpret_cast(address); + if (!local->sin_port) local->sin_port = htons(f.next_port++); + return 0; +} +int __wrap_rdma_listen(rdma_cm_id* id, int) { + f.ids.at(id).listening = true; + return 0; +} +int __wrap_rdma_resolve_addr(rdma_cm_id* id, sockaddr*, sockaddr* destination, + int) { + *reinterpret_cast(&id->route.addr.dst_addr) = + *reinterpret_cast(destination); + Event(id, RDMA_CM_EVENT_ADDR_RESOLVED); + return 0; +} +int __wrap_rdma_resolve_route(rdma_cm_id* id, int) { + Event(id, RDMA_CM_EVENT_ROUTE_RESOLVED); + return 0; +} +int __wrap_rdma_connect(rdma_cm_id* id, rdma_conn_param* param) { + EXPECT_EQ(id->qp, nullptr); + EXPECT_NE(param->qp_num, 0U); + if (f.suppress_connect) return 0; + rdma_cm_id* listener = nullptr; + for (const auto& [key, value] : f.ids) + if (value.listening && rdma_get_src_port(key) == rdma_get_dst_port(id)) { + listener = key; + break; + } + if (!listener) { + Event(id, RDMA_CM_EVENT_REJECTED, nullptr, nullptr, 0, ECONNREFUSED); + return 0; + } + rdma_cm_id* accepted; + __wrap_rdma_create_id(listener->channel, &accepted, nullptr, RDMA_PS_TCP); + f.ids.at(accepted).peer = id; + f.ids.at(id).peer = accepted; + Event(accepted, RDMA_CM_EVENT_CONNECT_REQUEST, listener, param->private_data, + param->private_data_len); + return 0; +} +int __wrap_rdma_accept(rdma_cm_id* id, rdma_conn_param* param) { + EXPECT_EQ(id->qp, nullptr); + EXPECT_NE(param->qp_num, 0U); + Event(f.ids.at(id).peer, RDMA_CM_EVENT_CONNECT_RESPONSE, nullptr, + param->private_data, param->private_data_len); + return 0; +} +int __wrap_rdma_establish(rdma_cm_id* id) { + if (f.suppress_establish) return 0; + Event(f.ids.at(id).peer, RDMA_CM_EVENT_ESTABLISHED); + return 0; +} +int __wrap_rdma_disconnect(rdma_cm_id* id) { + if (auto* peer = f.ids.at(id).peer) Event(peer, RDMA_CM_EVENT_DISCONNECTED); + return 0; +} +int __wrap_rdma_reject(rdma_cm_id* id, const void*, std::uint8_t) { + if (auto* peer = f.ids.at(id).peer) + Event(peer, RDMA_CM_EVENT_REJECTED, nullptr, nullptr, 0, ECONNREFUSED); + return 0; +} +int __wrap_rdma_get_cm_event(rdma_event_channel* channel, + rdma_cm_event** output) { + auto& queue = f.channels.at(channel); + if (queue.empty()) return Empty(); + Consume(channel->fd); + auto event = std::move(queue.front()); + queue.pop_front(); + *output = &event->event; + f.delivered.emplace(*output, std::move(event)); + return 0; +} +int __wrap_rdma_ack_cm_event(rdma_cm_event* event) { + EXPECT_EQ(f.delivered.erase(event), 1U); + return 0; +} +int __wrap_rdma_init_qp_attr(rdma_cm_id*, ibv_qp_attr*, int* mask) { + *mask = IBV_QP_STATE; + return 0; +} +const char* __wrap_ibv_get_device_name(ibv_device*) { return "fake0"; } +ibv_context* __wrap_ibv_open_device(ibv_device* dev) { + auto* context = new ibv_context{}; + context->device = dev; + context->async_fd = EventFd(); + context->ops.poll_cq = PollCq; + context->ops.req_notify_cq = ArmCq; + f.contexts.emplace(context, std::deque{}); + return context; +} +int __wrap_ibv_close_device(ibv_context* context) { + EXPECT_TRUE(f.contexts.at(context).empty()); + f.contexts.erase(context); + close(context->async_fd); + delete context; + return 0; +} +int __wrap_ibv_query_device(ibv_context*, ibv_device_attr* attr) { + *attr = {}; + attr->max_cqe = 65536; + attr->max_qp_wr = 8192; + return 0; +} +ibv_pd* __wrap_ibv_alloc_pd(ibv_context* context) { + auto* pd = new ibv_pd{}; + pd->context = context; + f.pds.insert(pd); + return pd; +} +int __wrap_ibv_dealloc_pd(ibv_pd* pd) { + EXPECT_EQ(f.pds.erase(pd), 1U); + delete pd; + return 0; +} +ibv_comp_channel* __wrap_ibv_create_comp_channel(ibv_context* context) { + auto* channel = new ibv_comp_channel{}; + channel->context = context; + channel->fd = EventFd(); + f.completion_channels.emplace(channel, std::deque{}); + return channel; +} +int __wrap_ibv_destroy_comp_channel(ibv_comp_channel* channel) { + EXPECT_TRUE(f.completion_channels.at(channel).empty()); + f.completion_channels.erase(channel); + close(channel->fd); + delete channel; + return 0; +} +ibv_cq* __wrap_ibv_create_cq(ibv_context* context, int cqe, void* opaque, + ibv_comp_channel* channel, int) { + auto* cq = new ibv_cq{}; + cq->context = context; + cq->cq_context = opaque; + cq->cqe = cqe; + cq->channel = channel; + f.cqs.emplace(cq, Cq{}); + return cq; +} +int __wrap_ibv_destroy_cq(ibv_cq* cq) { + EXPECT_EQ(f.cqs.erase(cq), 1U); + delete cq; + return 0; +} +ibv_qp* __wrap_ibv_create_qp(ibv_pd* pd, ibv_qp_init_attr* attr) { + if (std::exchange(f.fail_qp, false)) { + errno = ENOMEM; + return nullptr; + } + EXPECT_EQ(attr->qp_type, IBV_QPT_RC); + EXPECT_EQ(pd->context, attr->send_cq->context); + EXPECT_EQ(pd->context, attr->recv_cq->context); + attr->cap.max_send_wr += f.qp_extra; + auto* qp = new ibv_qp{}; + qp->pd = pd; + qp->context = pd->context; + qp->qp_num = f.next_qp++; + qp->qp_type = attr->qp_type; + qp->send_cq = attr->send_cq; + qp->recv_cq = attr->recv_cq; + f.qps.insert(qp); + return qp; +} +int __wrap_ibv_modify_qp(ibv_qp* qp, ibv_qp_attr* attr, int) { + qp->state = attr->qp_state; + return 0; +} +int __wrap_ibv_destroy_qp(ibv_qp* qp) { + EXPECT_EQ(f.qps.erase(qp), 1U); + delete qp; + return 0; +} +int __wrap_ibv_get_cq_event(ibv_comp_channel* channel, ibv_cq** cq, + void** opaque) { + auto& queue = f.completion_channels.at(channel); + if (queue.empty()) return Empty(); + Consume(channel->fd); + *cq = queue.front(); + *opaque = (*cq)->cq_context; + queue.pop_front(); + return 0; +} +void __wrap_ibv_ack_cq_events(ibv_cq*, unsigned count) { f.cq_acks += count; } +int __wrap_ibv_get_async_event(ibv_context* context, ibv_async_event* event) { + auto& queue = f.contexts.at(context); + if (queue.empty()) return Empty(); + Consume(context->async_fd); + *event = queue.front(); + queue.pop_front(); + return 0; +} +void __wrap_ibv_ack_async_event(ibv_async_event*) { ++f.async_acks; } +} // extern "C" diff --git a/tests/rdma_fake.h b/tests/rdma_fake.h new file mode 100644 index 0000000..bb3e8a4 --- /dev/null +++ b/tests/rdma_fake.h @@ -0,0 +1,29 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include + +#include + +namespace rdma_fake { +void Reset(); +void CheckEmpty(); +void SuppressConnect(bool suppress); +void SuppressEstablish(bool suppress); +void Cm(rdma_cm_id* id, rdma_cm_event_type event, int status = 0); +void RoundQp(unsigned extra); +void FailNextQp(); +void FailNextNotify(); +void Completion(ibv_cq* cq, std::uint32_t qp, + ibv_wc_status status = IBV_WC_SUCCESS); +void Async(ibv_context* context, ibv_event_type type, ibv_qp* qp = nullptr); +// Inject a CQE exactly when notification is requested, before the CQ recheck. +void CompleteOnArm(ibv_cq* cq, std::uint32_t qp); +int CqPolls(ibv_cq* cq); +int Notifications(ibv_cq* cq); +int AckedCqEvents(); +int AckedAsyncEvents(); +int LiveQps(); +int LiveIds(); +} // namespace rdma_fake diff --git a/tests/rdma_rc_test.cpp b/tests/rdma_rc_test.cpp new file mode 100644 index 0000000..5232a97 --- /dev/null +++ b/tests/rdma_rc_test.cpp @@ -0,0 +1,590 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../src/rdma/rc_internal.h" +#include "bycorf/rdma/rc.h" +#include "bycorf/runtime/worker.h" +#ifdef BYCORF_RDMA_FAKE +#include "bycorf/net/server.h" +#include "rdma_fake.h" +#define RcLoop RcLoopFake +#endif + +namespace bycorf::rdma::detail { +class RcTestPeer { + public: + static LocalSharedPtr State(RcConnection& c) { return c.state_; } + static LocalSharedPtr State(RcListener& l) { return l.state_; } +}; +} // namespace bycorf::rdma::detail +namespace { +using namespace bycorf; +using namespace bycorf::rdma; +using Peer = bycorf::rdma::detail::RcTestPeer; + +template +Task Collect(Task task, std::optional* result) { + result->emplace(co_await std::move(task)); + co_return absl::OkStatus(); +} +class RcLoop : public testing::Test { + protected: + void SetUp() override { +#ifdef BYCORF_RDMA_FAKE + rdma_fake::Reset(); + ip = "192.0.2.1"; +#else + ip = std::getenv("BYCORF_RDMA_TEST_IP"); + if (!ip || !*ip) GTEST_SKIP() << "Set BYCORF_RDMA_TEST_IP to opt in"; +#endif + cc.mailbox(0).wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(cc.mailbox(0).wake_fd_, 0); + worker.BindCrossCore(0, &cc); + SetThisWorker(0, &cc, &worker); + ASSERT_TRUE(worker.Init({.recv_buffer_count_ = 0}).ok()); + alarm(15); + initialized = true; + } + void TearDown() override { + if (!initialized) return; + alarm(0); + worker.Shutdown(); + worker.DestroyDetachedTasks(); + MutableThisWorker() = {}; + close(cc.mailbox(0).wake_fd_); +#ifdef BYCORF_RDMA_FAKE + rdma_fake::CheckEmpty(); +#endif + } + template + void Until(Predicate done) { + while (!done()) ASSERT_TRUE(worker.RunOnce(true)); + } + template + T Run(Task task) { + std::optional result; + worker.Spawn(Collect(std::move(task), &result)); + while (!result) worker.RunOnce(true); + return std::move(*result); + } + absl::StatusOr> Pair( + RcOptions options = {}) { + auto listener = RcListener::Listen(worker, ip, 0, options); + if (!listener.ok()) return listener.status(); + auto client = + Run(RcConnection::Connect(worker, ip, listener->port(), options)); + if (!client.ok()) return client.status(); + auto server = Run(listener->Accept()); + if (!server.ok()) return server.status(); + auto status = Run(listener->Close()); + if (!status.ok()) return status; + return std::pair(std::move(*client), std::move(*server)); + } + const char* ip = nullptr; + bool initialized = false; + CrossCore cc{1}; + Worker worker; +}; + +TEST_F(RcLoop, IdleConnectAcceptPrivateDataAndSharedCq) { + const std::array data{std::byte{0x31}, std::byte{0x72}}; + auto listening = RcListener::Listen(worker, ip, 0, {}, data); + ASSERT_TRUE(listening.ok()) << listening.status(); + auto listener = std::move(*listening); + std::optional> connected, accepted; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener.port(), {}, data), + &connected)); + worker.Spawn(Collect(listener.Accept(), &accepted)); + Until([&] { return connected && accepted; }); + ASSERT_TRUE(connected->ok()) << connected->status(); + ASSERT_TRUE(accepted->ok()) << accepted->status(); + auto client = std::move(**connected), server = std::move(**accepted); + EXPECT_TRUE(client.IsOpen()); + EXPECT_TRUE(server.IsOpen()); + ASSERT_GE(client.peer_private_data().size(), data.size()); + ASSERT_GE(server.peer_private_data().size(), data.size()); + EXPECT_TRUE( + std::equal(data.begin(), data.end(), client.peer_private_data().begin())); + EXPECT_TRUE( + std::equal(data.begin(), data.end(), server.peer_private_data().begin())); + EXPECT_EQ(client.protection_domain().native_pd(), + server.protection_domain().native_pd()); + auto a = Peer::State(client), b = Peer::State(server); + EXPECT_EQ(a->device->cq, b->device->cq); + EXPECT_EQ(a->qp->qp_type, IBV_QPT_RC); + EXPECT_EQ(a->qp->pd, client.protection_domain().native_pd()); + EXPECT_NE(a->id->verbs, a->qp->context); + auto device = a->device; + EXPECT_EQ(device->reserved, a->budget + b->budget); + EXPECT_TRUE(Run(listener.Close()).ok()); + EXPECT_TRUE(server.IsOpen()); + EXPECT_TRUE(Run(client.Close()).ok()); + EXPECT_TRUE(a->closed); + EXPECT_EQ(a->id, nullptr); + EXPECT_EQ(a->qp, nullptr); + Until([&] { return b->closed; }); + EXPECT_FALSE(server.IsOpen()); + EXPECT_FALSE(server.status().ok()); + EXPECT_EQ(device->reserved, 0); + EXPECT_TRUE(Run(server.Close()).ok()); +} + +TEST_F(RcLoop, CloseWakesAcceptAndShutdownRetiresPolls) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()) << listener.status(); + std::optional> accepted; + worker.Spawn(Collect(listener->Accept(), &accepted)); + worker.RunOnce(false); + auto state = Peer::State(*listener); + auto engine = state->engine.lock(); + ASSERT_TRUE(engine->cm_poll.armed); + EXPECT_TRUE(Run(listener->Close()).ok()); + Until([&] { return accepted.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(accepted->status())); + engine->BeginShutdown(); + EXPECT_TRUE(engine->cm_poll.armed) + << "cancel request must retain the poll tag"; + worker.Shutdown(); + EXPECT_FALSE(engine->cm_poll.armed); + EXPECT_EQ(engine->cm, nullptr); +} + +TEST_F(RcLoop, RefusedConnectionReportsCmError) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + const auto port = listener->port(); + ASSERT_TRUE(Run(listener->Close()).ok()); + auto result = Run(RcConnection::Connect(worker, ip, port)); + EXPECT_FALSE(result.ok()); + EXPECT_NE(std::string(result.status().message()).find("RDMA_CM_EVENT_"), + std::string::npos) + << result.status(); +} + +TEST_F(RcLoop, SeparateWorkersOwnDistinctContextsAndAcceptedConnections) { + CrossCore other_cc{1}; + Worker other; + other_cc.mailbox(0).wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(other_cc.mailbox(0).wake_fd_, 0); + other.BindCrossCore(0, &other_cc); + std::optional> client, server; + auto listener = RcListener::Listen(worker, ip, 0); + // On assertion failure quiesce both workers before result/Task destruction. + struct Cleanup { + Worker& a; + Worker& b; + CrossCore& ca; + CrossCore& cb; + ~Cleanup() { + SetThisWorker(0, &cb, &b); + b.Shutdown(); + b.DestroyDetachedTasks(); + close(cb.mailbox(0).wake_fd_); + SetThisWorker(0, &ca, &a); + a.Shutdown(); + a.DestroyDetachedTasks(); + } + } cleanup{worker, other, cc, other_cc}; + ASSERT_TRUE(listener.ok()); + worker.Spawn(Collect(listener->Accept(), &server)); + SetThisWorker(0, &other_cc, &other); + ASSERT_TRUE(other.Init({.recv_buffer_count_ = 0}).ok()); + other.Spawn( + Collect(RcConnection::Connect(other, ip, listener->port()), &client)); + while (!client || !server) { + SetThisWorker(0, &cc, &worker); + worker.RunOnce(false); + SetThisWorker(0, &other_cc, &other); + other.RunOnce(false); + } + ASSERT_TRUE(client->ok()) << client->status(); + ASSERT_TRUE(server->ok()) << server->status(); + auto a = Peer::State(**client), b = Peer::State(**server); + EXPECT_EQ(a->engine.lock()->worker, &other); + EXPECT_EQ(b->engine.lock()->worker, &worker); + EXPECT_NE(a->qp->context, b->qp->context); + EXPECT_NE(a->device->cq, b->device->cq); + SetThisWorker(0, &cc, &worker); + worker.Shutdown(); + EXPECT_TRUE(b->closed); + SetThisWorker(0, &other_cc, &other); + while (!a->closed) other.RunOnce(true); + SetThisWorker(0, &cc, &worker); +} + +TEST_F(RcLoop, ConcurrentAcceptFailsAndCloseWakesOriginalWaiter) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + std::optional> first; + worker.Spawn(Collect(listener->Accept(), &first)); + worker.RunOnce(false); + auto second = Run(listener->Accept()); + EXPECT_TRUE(absl::IsFailedPrecondition(second.status())); + EXPECT_TRUE(Run(listener->Close()).ok()); + Until([&] { return first.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(first->status())); +} + +TEST_F(RcLoop, InvalidOptionsAndPrivateDataAreRejected) { + RcOptions options; + options.connect_timeout = std::chrono::milliseconds(0); + EXPECT_TRUE(absl::IsInvalidArgument( + Run(RcConnection::Connect(worker, ip, 123, options)).status())); + options = {}; + options.send_wr = 0; + EXPECT_TRUE(absl::IsInvalidArgument( + RcListener::Listen(worker, ip, 0, options).status())); + std::array oversized{}; + EXPECT_TRUE(absl::IsInvalidArgument( + RcListener::Listen(worker, ip, 0, {}, oversized).status())); + EXPECT_TRUE(absl::IsInvalidArgument( + Run(RcConnection::Connect(worker, "hostname", 123)).status())); +} + +#ifdef BYCORF_RDMA_FAKE +TEST(RcThreadOwnership, ServerShutdownReclaimsSuspendedAcceptOnOwnerThread) { + rdma_fake::Reset(); + class ListeningService final : public Service { + public: + std::atomic entered{false}; + void Prepare(unsigned) override {} + void Stop() noexcept override {} + Task Run(Worker& worker, ServiceContext) override { + auto listener = RcListener::Listen(worker, "192.0.2.1", 0); + if (!listener.ok()) co_return listener.status(); + entered.store(true, std::memory_order_release); + // Shutdown destroys this suspended frame and its local listener on the + // worker, before the control thread destroys Server and Worker objects. + auto accepted = co_await listener->Accept(); + co_return accepted.status(); + } + } service; + { + Server server; + server.AddService(&service); + ASSERT_TRUE(server + .Start({.thread_count_ = 1, + .pin_workers_ = false, + .recv_buffer_count_ = 0}) + .ok()); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!service.entered.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + EXPECT_TRUE(service.entered.load(std::memory_order_acquire)); + server.RequestStop(); + server.WaitUntilStopped(); + EXPECT_EQ(server.exit_code(), 0); + } + rdma_fake::CheckEmpty(); +} + +TEST_F(RcLoop, QueuedAcceptPreservesOrderAndRetainsFailedResults) { + RcOptions options; + options.send_wr = options.recv_wr = 4; + auto listener = RcListener::Listen(worker, ip, 0, options); + ASSERT_TRUE(listener.ok()); + auto state = Peer::State(*listener); + auto engine = state->engine.lock(); + std::vector clients; + std::vector> queued; + // Queue a burst before Accept so both the child table and FIFO must grow. + for (unsigned i = 0; i < 12; ++i) { + const std::array data{static_cast(i)}; + auto client = + Run(RcConnection::Connect(worker, ip, listener->port(), options, data)); + ASSERT_TRUE(client.ok()) << client.status(); + clients.push_back(std::move(*client)); + Until([&] { return state->ready.size() == i + 1; }); + queued.push_back(state->children.at(state->ready[i])); + } + { + auto failed = queued[3].lock(); + ASSERT_TRUE(failed); + const auto id = failed->id; + rdma_fake::Cm(id, RDMA_CM_EVENT_DISCONNECTED); + Until([&] { return failed->closed; }); + EXPECT_FALSE(engine->connections.contains(id)); + } + // Only the listener retains this retired state until Accept reads its error. + EXPECT_FALSE(queued[3].expired()); + std::vector accepted; + for (unsigned i = 0; i < queued.size(); ++i) { + auto connection = Run(listener->Accept()); + if (i == 3) { + EXPECT_TRUE(absl::IsUnavailable(connection.status())); + EXPECT_TRUE(queued[i].expired()); + } else { + ASSERT_TRUE(connection.ok()) << connection.status(); + ASSERT_FALSE(connection->peer_private_data().empty()); + EXPECT_EQ(connection->peer_private_data().front(), + static_cast(i)); + accepted.push_back(std::move(*connection)); + } + } + EXPECT_TRUE(state->ready.empty()); + EXPECT_TRUE(state->children.empty()); + EXPECT_TRUE(Run(listener->Close()).ok()); + for (const auto& connection : accepted) EXPECT_TRUE(connection.IsOpen()); +} + +TEST_F(RcLoop, LazyAcceptRetainsStateAfterListenerHandleDestruction) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + LocalWeakPtr state = Peer::State(*listener); + auto accepting = listener->Accept(); + *listener = RcListener{}; + EXPECT_FALSE(state.expired()); + EXPECT_TRUE(absl::IsCancelled(Run(std::move(accepting)).status())); + EXPECT_TRUE(state.expired()); +} + +TEST_F(RcLoop, ListenerCloseReleasesUnacceptedReadyStates) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto state = Peer::State(*listener); + auto client = Run(RcConnection::Connect(worker, ip, listener->port())); + ASSERT_TRUE(client.ok()) << client.status(); + Until([&] { return !state->ready.empty(); }); + LocalWeakPtr queued = + state->children.at(state->ready.front()); + EXPECT_TRUE(Run(listener->Close()).ok()); + EXPECT_TRUE(state->ready.empty()); + EXPECT_TRUE(state->children.empty()); + EXPECT_TRUE(queued.expired()); + EXPECT_TRUE(absl::IsCancelled(Run(listener->Accept()).status())); +} + +TEST_F(RcLoop, NotifyRecheckRaceAndRepeatedIdleNotifications) { + auto pair = Pair(); + ASSERT_TRUE(pair.ok()) << pair.status(); + auto state = Peer::State(pair->first); + auto* cq = state->device->cq; + const int polls = rdma_fake::CqPolls(cq); + rdma_fake::CompleteOnArm(cq, state->qp->qp_num); + rdma_fake::Completion(cq, state->qp->qp_num); + Until([&] { return rdma_fake::AckedCqEvents() >= 2; }); + EXPECT_GE(rdma_fake::CqPolls(cq), polls + 2); + EXPECT_EQ(rdma_fake::Notifications(cq), 2); + rdma_fake::Completion(cq, state->qp->qp_num); + Until([&] { return rdma_fake::AckedCqEvents() == 3; }); + EXPECT_TRUE(pair->first.IsOpen()); +} + +TEST_F(RcLoop, TimeoutRetiresIdQpAndTimerBeforeReturn) { + rdma_fake::SuppressConnect(true); + RcOptions options; + options.connect_timeout = std::chrono::milliseconds(10); + auto result = Run(RcConnection::Connect(worker, ip, 23456, options)); + EXPECT_TRUE(absl::IsDeadlineExceeded(result.status())) << result.status(); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + rdma_fake::SuppressConnect(false); + EXPECT_TRUE(Pair().ok()); +} + +TEST_F(RcLoop, SharedCqCapacityReleasedOnlyAfterQpRetirement) { + RcOptions options; + options.send_wr = options.recv_wr = 4; + options.cq_entries = 16; + auto first = Pair(options); + ASSERT_TRUE(first.ok()) << first.status(); + auto device = Peer::State(first->first)->device; + EXPECT_EQ(device->reserved, 16); + auto exhausted = Run(RcConnection::Connect(worker, ip, 12345, options)); + EXPECT_TRUE(absl::IsResourceExhausted(exhausted.status())) + << exhausted.status(); + EXPECT_EQ(device->reserved, 16); + EXPECT_TRUE(Run(first->first.Close()).ok()); + EXPECT_TRUE(Run(first->second.Close()).ok()); + EXPECT_EQ(device->reserved, 0); + EXPECT_TRUE(Pair(options).ok()); +} + +TEST_F(RcLoop, ProviderRoundedQpCapacityCannotOvercommitCq) { + rdma_fake::RoundQp(9); + RcOptions options; + options.send_wr = options.recv_wr = 4; + options.cq_entries = 16; + auto result = Run(RcConnection::Connect(worker, ip, 12345, options)); + EXPECT_TRUE(absl::IsResourceExhausted(result.status())) << result.status(); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_EQ(rdma_fake::LiveIds(), 0); +} + +TEST_F(RcLoop, FailedQpCreationUnwindsAndCanRetry) { + rdma_fake::FailNextQp(); + auto result = Run(RcConnection::Connect(worker, ip, 12345)); + EXPECT_FALSE(result.ok()); + EXPECT_NE(std::string(result.status().message()).find("ibv_create_qp"), + std::string::npos); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_TRUE(Pair().ok()); +} + +TEST_F(RcLoop, FailedNotificationSetupRetainsNoConnection) { + rdma_fake::FailNextNotify(); + auto result = Run(RcConnection::Connect(worker, ip, 12345)); + EXPECT_FALSE(result.ok()); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, CqFailureTerminatesAllSharingConnectionsAfterAck) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_CQ_ERR, state->qp); + Until([&] { return !first->first.IsOpen() && !second->first.IsOpen(); }); + EXPECT_EQ(rdma_fake::AckedAsyncEvents(), 1); + EXPECT_FALSE(first->second.IsOpen()); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_NE(std::string(second->first.status().message()) + .find(ibv_event_type_str(IBV_EVENT_CQ_ERR)), + std::string::npos); +} + +TEST_F(RcLoop, DeviceRemovalWithStatusFailsEveryAffectedConnection) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Cm(state->id, RDMA_CM_EVENT_DEVICE_REMOVAL, EIO); + Until([&] { return !first->first.IsOpen() && !second->first.IsOpen(); }); + EXPECT_FALSE(first->second.IsOpen()); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, QpFailureLeavesOtherConnectionsUsable) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_QP_FATAL, state->qp); + Until([&] { return Peer::State(first->second)->closed; }); + EXPECT_EQ(rdma_fake::AckedAsyncEvents(), 1); + EXPECT_TRUE(second->first.IsOpen()); + EXPECT_TRUE(second->second.IsOpen()); +} + +TEST_F(RcLoop, WorkerShutdownRetainsSharedPollsUntilRingQuiescence) { + auto pair = Pair(); + ASSERT_TRUE(pair.ok()); + auto state = Peer::State(pair->first); + auto engine = state->engine.lock(); + auto device = state->device; + rdma_fake::Completion(device->cq, state->qp->qp_num); + engine->BeginShutdown(); + EXPECT_TRUE(engine->cm_poll.armed); + EXPECT_TRUE(device->completion_poll.armed); + EXPECT_TRUE(device->async_poll.armed); + EXPECT_NE(device->cq, nullptr); + worker.Shutdown(); + EXPECT_FALSE(engine->cm_poll.armed); + EXPECT_FALSE(device->completion_poll.armed); + EXPECT_FALSE(device->async_poll.armed); + EXPECT_TRUE(state->closed); + EXPECT_EQ(rdma_fake::AckedCqEvents(), 1); +} + +TEST_F(RcLoop, ListenerCloseJoinsHandshakingChildren) { + rdma_fake::SuppressEstablish(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto client = Run(RcConnection::Connect(worker, ip, listener->port())); + ASSERT_TRUE(client.ok()) << client.status(); + auto state = Peer::State(*listener); + ASSERT_EQ(state->children.size(), 1U); + auto pending = state->children.begin()->second; + ASSERT_TRUE(pending->timer.armed); + EXPECT_TRUE(Run(listener->Close()).ok()); + EXPECT_FALSE(pending->timer.armed); + EXPECT_TRUE(pending->closed); + EXPECT_EQ(pending->id, nullptr); + EXPECT_TRUE(state->children.empty()); + Until([&] { return Peer::State(*client)->closed; }); +} + +TEST_F(RcLoop, CancellationKeepsConnectionUntilTimerTerminalCompletion) { + rdma_fake::SuppressConnect(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto engine = Peer::State(*listener)->engine.lock(); + std::optional> result; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener->port()), &result)); + Until([&] { return rdma_fake::LiveQps() > 0; }); + auto state = engine->connections.begin()->second; + engine->Close(state, absl::CancelledError("test cancel")); + EXPECT_TRUE(state->timer.armed); + EXPECT_NE(state->id, nullptr); + EXPECT_FALSE(state->closed); + // An event queued before the original poll retires must not resurrect it. + rdma_fake::Cm(state->id, RDMA_CM_EVENT_CONNECT_RESPONSE); + Until([&] { return result.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(result->status())); + EXPECT_TRUE(state->closed); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, ShutdownWithSuspendedConnectAndAcceptReclaimsFrames) { + rdma_fake::SuppressConnect(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto engine = Peer::State(*listener)->engine.lock(); + std::optional> connected, accepted; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener->port()), &connected)); + worker.Spawn(Collect(listener->Accept(), &accepted)); + Until([&] { return rdma_fake::LiveQps() > 0; }); + auto state = engine->connections.begin()->second; + worker.Shutdown(); + EXPECT_TRUE(state->closed); + EXPECT_FALSE(state->timer.armed); + EXPECT_TRUE(engine->connections.empty()); + worker.DestroyDetachedTasks(); + EXPECT_FALSE(state->connected.waiter); + EXPECT_FALSE(Peer::State(*listener)->accepted.waiter); + EXPECT_EQ(rdma_fake::LiveIds(), 0); +} + +TEST_F(RcLoop, DeviceFatalAndNotifyRearmFailureTerminateConnections) { + auto first = Pair(); + ASSERT_TRUE(first.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_DEVICE_FATAL); + Until([&] { return state->closed; }); + EXPECT_FALSE(first->second.IsOpen()); + // Drive terminal cancelled polls so failed device resources can retire. + auto engine = state->engine.lock(); + Until([&] { return engine->devices.empty(); }); + auto second = Pair(); + ASSERT_TRUE(second.ok()) << second.status(); + auto state2 = Peer::State(second->first); + rdma_fake::FailNextNotify(); + rdma_fake::Completion(state2->device->cq, state2->qp->qp_num); + Until([&] { return state2->closed; }); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_NE( + std::string(second->first.status().message()).find("ibv_req_notify_cq"), + std::string::npos); +} +#endif +} // namespace