From 896d9a0208125102c5cfe273191290f53dfc074d Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 11 Nov 2026 15:59:48 +0000 Subject: [PATCH 1/5] feat(rdma): add RC connections and notification-driven event loop --- CMakeLists.txt | 22 +- README.md | 40 +- docs/architecture/README.md | 7 +- docs/architecture/io.md | 11 +- docs/architecture/rdma.md | 94 +++- docs/architecture/runtime.md | 13 +- include/bycorf/io/io_uring_backend.h | 4 + include/bycorf/io/rdma_backend.h | 27 +- include/bycorf/io/rdma_reactor.h | 26 + include/bycorf/rdma/rc.h | 111 ++++ include/bycorf/runtime/worker.h | 3 + src/io/io_uring_backend.cpp | 9 + src/rdma/rc.cpp | 199 +++++++ src/rdma/rc_engine.cpp | 761 +++++++++++++++++++++++++++ src/rdma/rc_internal.h | 160 ++++++ src/runtime/worker.cpp | 18 +- tests/io_uring_backpressure_test.cpp | 21 + tests/rdma_fake.cpp | 403 ++++++++++++++ tests/rdma_fake.h | 29 + tests/rdma_rc_test.cpp | 472 +++++++++++++++++ 20 files changed, 2394 insertions(+), 36 deletions(-) create mode 100644 include/bycorf/io/rdma_reactor.h create mode 100644 include/bycorf/rdma/rc.h create mode 100644 src/rdma/rc.cpp create mode 100644 src/rdma/rc_engine.cpp create mode 100644 src/rdma/rc_internal.h create mode 100644 tests/rdma_fake.cpp create mode 100644 tests/rdma_fake.h create mode 100644 tests/rdma_rc_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 97f9cdb..a7f3fc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,7 +186,8 @@ if (BYCORF_ENABLE_RDMA) find_package(PkgConfig REQUIRED) pkg_check_modules(BYCORF_IBVERBS REQUIRED IMPORTED_TARGET libibverbs) pkg_check_modules(BYCORF_RDMACM REQUIRED IMPORTED_TARGET librdmacm) - target_sources(bycorf PRIVATE src/io/rdma_memory.cpp) + target_sources(bycorf PRIVATE src/io/rdma_memory.cpp src/rdma/rc.cpp + src/rdma/rc_engine.cpp) target_link_libraries(bycorf PUBLIC PkgConfig::BYCORF_IBVERBS PkgConfig::BYCORF_RDMACM) endif() @@ -487,6 +488,25 @@ if (BYCORF_BUILD_TESTS) endforeach() bycorf_discover_tests(bycorf_rdma_memory_tests 30 "runtime;rdma;unit") + add_executable(bycorf_rdma_rc_fake_tests tests/rdma_rc_test.cpp tests/rdma_fake.cpp) + target_compile_definitions(bycorf_rdma_rc_fake_tests PRIVATE BYCORF_RDMA_FAKE=1) + target_link_libraries(bycorf_rdma_rc_fake_tests PRIVATE bycorf::core GTest::gtest_main) + foreach(symbol IN ITEMS rdma_create_event_channel rdma_destroy_event_channel + rdma_create_id rdma_destroy_id rdma_bind_addr rdma_listen rdma_resolve_addr + rdma_resolve_route rdma_connect rdma_accept rdma_establish rdma_disconnect + rdma_reject rdma_get_cm_event rdma_ack_cm_event rdma_init_qp_attr + ibv_get_device_name ibv_open_device ibv_close_device ibv_query_device + ibv_alloc_pd ibv_dealloc_pd ibv_create_comp_channel ibv_destroy_comp_channel + ibv_create_cq ibv_destroy_cq ibv_create_qp ibv_modify_qp ibv_destroy_qp + ibv_get_cq_event ibv_ack_cq_events ibv_get_async_event ibv_ack_async_event) + target_link_options(bycorf_rdma_rc_fake_tests PRIVATE "LINKER:--wrap=${symbol}") + endforeach() + bycorf_discover_tests(bycorf_rdma_rc_fake_tests 30 "runtime;rdma;stub") + + add_executable(bycorf_rdma_rc_tests tests/rdma_rc_test.cpp) + target_link_libraries(bycorf_rdma_rc_tests PRIVATE bycorf::core GTest::gtest_main) + bycorf_discover_tests(bycorf_rdma_rc_tests 30 "runtime;rdma;integration") + add_executable(bycorf_rdma_registration_tests tests/rdma_registration_test.cpp) target_link_libraries(bycorf_rdma_registration_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_rdma_registration_tests 30 "runtime;rdma;integration") diff --git a/README.md b/README.md index 3deb16e..c12bfcd 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ connections, and asynchronous I/O; applications provide services above - [Name](#name) - [Build and test](#build-and-test) -- [Optional RDMA registration](#optional-rdma-registration) +- [Optional RDMA connections and registration](#optional-rdma-connections-and-registration) - [Run the examples](#run-the-examples) - [Use Bycorf in a CMake project](#use-bycorf-in-a-cmake-project) - [Development](#development) @@ -179,10 +179,10 @@ Follow the [DPDK/SPDK build and test guide](docs/dpdk-prototype.md) for its additional dependencies, virtual-device checks, physical-device configuration, and experimental compatibility limits. -## Optional RDMA registration +## Optional RDMA connections and registration -`BYCORF_ENABLE_RDMA` enables the private worker facade and explicit memory -registration API. It defaults to off and is independent of +`BYCORF_ENABLE_RDMA` enables the private worker facade, RC connection API and +explicit memory registration. It defaults to off and is independent of `BYCORF_KERNEL_BYPASS`; enabling it requires libibverbs and librdmacm development packages, without DPDK/SPDK. On Ubuntu: @@ -190,7 +190,7 @@ packages, without DPDK/SPDK. On Ubuntu: sudo apt-get install --no-install-recommends libibverbs-dev librdmacm-dev cmake -S . -B build-rdma -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBYCORF_ENABLE_RDMA=ON cmake --build build-rdma --parallel -ctest --test-dir build-rdma -R '^bycorf_RdmaMemory\.' --output-on-failure +ctest --test-dir build-rdma -R '^bycorf_(RdmaMemory|RcLoopFake)\.' --output-on-failure ``` The device-free `RdmaMemory` suite injects verbs failures and verifies lazy @@ -206,9 +206,21 @@ BYCORF_RDMA_TEST_DEVICE=rxe0 ctest --test-dir build-rdma \ This opens the selected device, registers and deregisters memory, creates real QPs sharing a PD, and checks permissions, bounds, and mismatched PD rejection. -It requires access to the device's uverbs node and sufficient memlock. It does -not test a connected data-transfer or immediate-data protocol. No device is -opened by unused workers or test discovery. +It requires access to the device's uverbs node and sufficient memlock. For RC +connection tests on an existing Soft-RoCE interface, supply its local IP: + +```bash +BYCORF_RDMA_TEST_IP=192.0.2.1 ctest --test-dir build-rdma \ + -R '^bycorf_RcLoop\.' --output-on-failure +``` + +These exercise idle-worker connection events, private data, distinct worker +contexts, shared CQ/PD ownership, refused connections and close/accept races. +The device-free `RcLoopFake` cases additionally inject notification races, +provider capacity rounding, budget exhaustion, deadlines, CQ/device failures, +and late events during cancellation and shutdown. They use real io_uring with +link-time CM/verbs stubs. Data-transfer and immediate-data protocols remain +outside these tests. No device is opened by unused workers or test discovery. Use `Worker::OpenRdmaDevice(name)` or `AcquireRdmaDomain(owning_context)` on the owner worker, then `RegisterRdmaMemory(domain, backing_span, explicit_access)`. @@ -219,6 +231,18 @@ backing memory and must stop remote access before releasing registrations and freeing it. See [RDMA ownership](docs/architecture/rdma.md) for remote descriptors, permissions, CM context ownership, and the independent storage-buffer contract. +The RC API is in `bycorf/rdma/rc.h`. On the owning worker, call +`RcListener::Listen(worker, numeric_ip, port)` and await `listener.Accept()`, or +await `RcConnection::Connect(worker, numeric_ip, port)`. Pass `RcOptions` to set +QP/CQ capacities and the handshake timeout, and an optional byte span for CM +private data. Await `connection.Close()` or `listener.Close()` to join resource +retirement. Destruction only requests close. Use +`connection.protection_domain()` when registering memory for that connection. +An accepted connection stays on the listener's worker; moving a C++ handle +does not move worker ownership. Private data is limited to 56 bytes; CM may +report a padded peer-data length. The first connection fixes that worker/device +CQ capacity, and subsequent connections must request the same capacity. + ## Run the examples The default build enables `BYCORF_BUILD_EXAMPLES`. If it was disabled, set diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 874c894..0038402 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -23,8 +23,9 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging. | Module | Responsibility | Document | |---|---|---| | Runtime | Worker lifecycle, coroutine scheduling, sleep coordination, cross-worker and foreign ingress | [Runtime](runtime.md) | -| I/O | Worker-owned shared io_uring, network/storage adapters, optional RDMA registration and SPDK storage, shared DPDK environment | [I/O backends](io.md), [RDMA ownership](rdma.md) | +| I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | | Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) | +| RDMA | Worker-owned RC connections, CM/QP lifecycle, shared CQ notifications and memory registration | [RDMA](rdma.md) | | RPC | Framed request/reply protocol above streams | [RPC](rpc.md) | Each connection and its coroutines have one worker owner. Key ownership and @@ -38,8 +39,8 @@ worker before that coroutine uses its stream again. Startup selection independently activates network and storage backends and remains fixed for the process lifetime. Both default to io_uring even in a bypass-capable build. -`BYCORF_ENABLE_RDMA` independently enables the RDMA facade and registration -layer; resources are acquired only on explicit use. +`BYCORF_ENABLE_RDMA` independently enables the RDMA facade, RC connections and +registration layer; resources are acquired only on explicit use. The [prototype runbook](../dpdk-prototype.md) describes the DPDK configuration. Architecture documents describe the current core model and stable tradeoffs. diff --git a/docs/architecture/io.md b/docs/architecture/io.md index a4a317a..39f031e 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -44,11 +44,12 @@ Network adapters supply notification preparation, idle checks, deadlines, and cleanup to the [worker sleep protocol](runtime.md#sleep-and-wakeups). The worker enters the ring's blocking wait; adapters never block independently. -The ring outlives all facades. RDMA drops its weak resource cache on shutdown; -registration and operation handles retain their PD/context independently. The -worker initializes the ring first, stops storage and networking in that order, -then shuts the ring down. Storage dispatch remains -valid through shutdown completion draining; SPDK callbacks use the owning +The ring outlives all facades. RDMA stops admission before ring shutdown but +retains channel polls until the ring has quiesced; it then retires remaining +RDMA events and connection resources. Exported registration handles retain +their PD/context independently. The worker initializes the ring first, stops +storage and networking in that order, then shuts the ring down. Storage dispatch +remains valid through shutdown completion draining; SPDK callbacks use the owning worker's TLS to reach its storage adapter. Kernel registrations remain live until ring shutdown quiesces kernel access. diff --git a/docs/architecture/rdma.md b/docs/architecture/rdma.md index 0f17263..f2f24ae 100644 --- a/docs/architecture/rdma.md +++ b/docs/architecture/rdma.md @@ -14,14 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. --> -# RDMA registration and ownership +# RDMA connections, notifications and memory ownership `BYCORF_ENABLE_RDMA` adds an independent I/O capability backed by libibverbs and librdmacm. It defaults to off and does not select or require DPDK/SPDK. The private `RdmaBackend` facade reaches the worker's existing io_uring through `KernelBackend()` for channel readiness and cancellation. It creates no ring, polling thread, thread pool, or process-wide blocking lock. The worker -continues to own shared-ring submission and completion dispatch. +continues to own shared-ring submission, completion dispatch and sleeping. +`bycorf::rdma::RcListener` and `RcConnection` provide numeric-address RC +Listen/Accept/Connect/Close and CM private data independently of TCP streams. Construction does not enumerate or open devices. Explicit `OpenRdmaDevice` selects a named verbs device; `AcquireRdmaDomain` accepts an owning reference to @@ -37,8 +39,88 @@ independent of worker shutdown. Partial initialization unwinds in reverse order. PD and retain the domain until QP destruction. A registered MR can be used by multiple QPs in the same PD; it belongs to neither a particular QP nor an IMM (immediate-data) operation. Operation leases check both the QP's context and PD. -Connection management, QP posting and completion protocols are outside this -registration layer. +RC connections expose their actual protection domain for registration. A caller +must use that domain rather than assume a separately opened device handle has +the same PD. Data-posting and application completion protocols are outside the +connection API. + +## RC ownership and connection flow + +Each worker lazily owns one nonblocking CM channel and its ID registry. Each +listener and every connection accepted by it stay on that worker; a connection +returned by Accept survives listener close. Connect and accepted handshakes +have one absolute monotonic deadline covering address resolution, route +resolution and establishment. Submission backpressure does not restart it. +Arguments are copied before returning a lazy task. Private data is limited to +56 bytes in both directions; peer data is copied before acknowledging the CM +event, with the length reported by CM (which can include provider padding). +Listen binds immediately; Accept, Connect and Close are awaitable operations. + +Librdmacm shares its routing contexts process-wide. RC therefore opens a private +verbs context per worker on the device selected by CM, and uses CM's external-QP +mode. CM negotiates the path/QP number; the worker creates only RC QPs in its own +PD and applies CM-provided INIT/RTR/RTS attributes, then completes the active +handshake with `rdma_establish`. It does not replace CM's context or attach its +QP to the CM ID. QP, PD, MR, CQ and the verbs asynchronous event stream belong to +the same private context, with exactly one worker reading that event stream. +Other workers neither consume its events nor migrate its connections. + +Each private context shares one completion channel and fixed-capacity CQ. +Admission reserves the provider's actual send plus receive WR capacities for +each QP, including capacities rounded up by the provider. All send WRs are +signalled. A QP remains unconnected and has no posted work until its actual +budget fits; excess capacity fails admission instead of growing the CQ. +Reservations cover error/flush completions as well as successful completions, +and are returned only after QP destruction and CQ draining. + +## Notification and failure flow + +The CM channel, completion channel and private context's async-event fd are +nonblocking, with persistent one-shot readiness polls in the shared io_uring. +A CQ notification is requested before checking the CQ, including initial +activation. Each delivered CQ event is acknowledged, notification is requested +again, and the CQ is drained before rearming the fd poll. An event may remain +after its CQE was already consumed; it still needs acknowledgement and rearming. +Notifications stay enabled during busy and idle periods. This makes pending +work either visible in the CQ recheck or capable of waking an idle worker; +there is no separate polling thread or second ring. + +CM events are copied and acknowledged before dispatch can destroy an ID. +Failures retain the CM event name/raw status or the failing verbs operation. +QP errors terminate the affected connection; shared CQ errors, device fatal +errors and device removal terminate every connection on that device context. +Port errors terminate connections using the affected port. Event objects are +acknowledged before this fanout can destroy their referenced resources. A +failed CM channel stops all endpoints on its worker and rejects new admission. + +## Close and worker shutdown + +Public handles are move-only and used on their owner worker. Destruction +requests close; awaiting Close joins retirement. Closing stops admission, +disconnects the CM association and puts the external QP into ERR. The +connection retains its timer tag until the timer's terminal completion, even +when cancellation has already been accepted. It leaves the ID registry before +resource destruction and releases its CQ budget only after the QP is destroyed +and the CQ drained. The shared channel polls reference the worker's event +engine/device state, never a listener or connection object. + +Listener close terminates queued and handshaking children and wakes Accept; +already returned connections remain independent. One Accept and one Close +waiter per handle are supported. Waiters detach when their coroutine frames +are reclaimed. Failed device state is released only after its connections and +both fd polls have retired. A cancellation request or the cancel request's own +CQE is not a retirement boundary for the original fd poll. + +Worker shutdown first stops RDMA admission and requests cancellation while +retaining every poll tag and descriptor. It then quiesces the shared ring. +Only after ring shutdown confirms that no kernel operation can refer to those +tags does RDMA retire requests that did not receive callbacks, acknowledge +remaining events, destroy QPs/CM IDs, and release channels/CQs. Coroutine frames +are reclaimed afterward. Exported registration/PD handles retain their own +context independently. Unexpected destruction failures fail closed rather than +release resources that may still be referenced by the provider. + +## Registered memory `RegisteredMemory` registers caller-owned backing memory with explicit access permissions. Copies, registered slices, and operation leases retain the same @@ -67,4 +149,6 @@ access and lifetime requirements. Sources: `include/bycorf/io/rdma_backend.h`, `include/bycorf/io/rdma_memory.h`, `src/io/rdma_memory.cpp`, -`include/bycorf/runtime/worker.h`, `src/runtime/worker.cpp`. +`include/bycorf/rdma/rc.h`, `src/rdma/rc.cpp`, `src/rdma/rc_engine.cpp`, +`src/rdma/rc_internal.h`, `include/bycorf/runtime/worker.h`, +`src/runtime/worker.cpp`, `src/io/io_uring_backend.cpp`. diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index ad6ac73..c4edb18 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -53,7 +53,11 @@ notifications, rechecks I/O, mailbox and scheduler work, completes the mailbox wake-sequence handshake, and submits operations and peer wakes. After a final idle check it waits on the shared io_uring. Runnable work, a stop request, unsubmitted operations, or storage requiring polling prevent sleep. Network -deadlines and enabled connection-idle checks bound the wait. +deadlines and enabled connection-idle checks bound the wait. Optional RDMA CM, +CQ-notification and device-async polls are persistent wake sources on this same +ring; RDMA connection deadlines are absolute ring timeouts. CQ notifications +are always armed and rechecked, so RDMA needs no busy polling or additional +sleep-preparation phase. Cross-worker submissions enter destination mailboxes and resume coroutines on their original workers. The mailbox handshake makes racing publication either @@ -78,8 +82,11 @@ an empty selection means every worker. Prepare sees the full runtime count, while Run and FinalizeWorker execute only on selected workers. During shutdown, workers first leave their loops, then quiesce I/O and reclaim coroutine frames on their owning native threads. Service finalizers run after all worker frames -are reclaimed. A worker that fails initialization contributes to both teardown -barriers so initialized peers can finish. Temporary I/O submission shortages +are reclaimed. RDMA uses a two-phase shutdown: stop admission before the ring +is quiesced, then release retained polls/events and connection resources after +kernel access has ended. This also covers accepted operations whose callbacks +are suppressed by ring shutdown. A worker that fails initialization contributes +to both teardown barriers so initialized peers can finish. Temporary I/O submission shortages are retried by the worker loop. A permanent submission error, notification preparation or cleanup failure, or shared-ring wait failure stops peer workers before entering those barriers and yields a nonzero exit. Services outlive diff --git a/include/bycorf/io/io_uring_backend.h b/include/bycorf/io/io_uring_backend.h index 7fe46f0..c71e146 100644 --- a/include/bycorf/io/io_uring_backend.h +++ b/include/bycorf/io/io_uring_backend.h @@ -135,6 +135,10 @@ class IoUringBackend { absl::Status SubmitFdatasync(FixedFile file, IoCompletion* tag); absl::Status SubmitTimeout(const __kernel_timespec& timeout, IoCompletion* tag); + // CLOCK_MONOTONIC absolute deadline. Admission/submission backpressure must + // not restart the budget. Caller retains timespec/tag until terminal CQE. + absl::Status SubmitDeadline(const __kernel_timespec& deadline, + IoCompletion* tag); // Observe a host descriptor (e.g. a DPDK RX notification) through the same // CQ as storage and MSG_RING. The caller retains tag until completion/cancel. absl::Status SubmitPoll(int fd, unsigned events, IoCompletion* tag); diff --git a/include/bycorf/io/rdma_backend.h b/include/bycorf/io/rdma_backend.h index cf5519f..7bc90bc 100644 --- a/include/bycorf/io/rdma_backend.h +++ b/include/bycorf/io/rdma_backend.h @@ -20,6 +20,7 @@ #if BYCORF_ENABLE_RDMA #include "bycorf/io/io_uring_backend.h" #include "bycorf/io/rdma_memory.h" +#include "bycorf/io/rdma_reactor.h" namespace bycorf { // Private CRTP facade: the owner supplies its existing KernelBackend(). Neither @@ -55,9 +56,28 @@ class RdmaBackend { absl::Status CancelRdmaEventPoll(IoCompletion* tag) { return Kernel().SubmitCancel(tag); } - // Drops only cache entries. Outstanding handles continue to own resources; - // callers must quiesce QPs/remote access before releasing those handles. - void Shutdown() noexcept { resources_.Shutdown(); } + // Absolute CLOCK_MONOTONIC deadline; caller retains deadline/tag until its + // terminal completion, including deferred submission and cancellation. + absl::Status SubmitRdmaDeadline(const __kernel_timespec& deadline, + IoCompletion* tag) { + return Kernel().SubmitDeadline(deadline, tag); + } + // Stop admission before the ring shuts down. Tags/resources remain retained + // until terminal completions or AfterRingShutdown confirms kernel quiescence. + void Shutdown() { rdma::detail::BeginShutdown(engine_); } + void AfterRingShutdown() { + rdma::detail::AfterRingShutdown(engine_); + engine_.reset(); + resources_.Shutdown(); + } + absl::StatusOr> GetRdmaEngine() { + if (!engine_) { + auto created = rdma::detail::CreateEngine(static_cast(*this)); + if (!created.ok()) return created.status(); + engine_ = *created; + } + return engine_; + } private: friend Owner; @@ -69,6 +89,7 @@ class RdmaBackend { return static_cast(*this).KernelBackend(); } detail::RdmaResources resources_; + std::shared_ptr engine_; }; } // namespace bycorf #endif // BYCORF_ENABLE_RDMA diff --git a/include/bycorf/io/rdma_reactor.h b/include/bycorf/io/rdma_reactor.h new file mode 100644 index 0000000..4d807ee --- /dev/null +++ b/include/bycorf/io/rdma_reactor.h @@ -0,0 +1,26 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#if BYCORF_ENABLE_RDMA +#include + +#include "absl/status/statusor.h" + +namespace bycorf { +class Worker; +namespace rdma { +class RcConnection; +class RcListener; +namespace detail { +class Engine; +// Used only by the private worker facade. Shutdown has two phases because +// io_uring drops outstanding tags without callbacks when the ring is destroyed. +absl::StatusOr> CreateEngine(Worker& worker); +void BeginShutdown(const std::shared_ptr& engine); +void AfterRingShutdown(const std::shared_ptr& engine); +} // namespace detail +} // namespace rdma +} // namespace bycorf +#endif diff --git a/include/bycorf/rdma/rc.h b/include/bycorf/rdma/rc.h new file mode 100644 index 0000000..e774261 --- /dev/null +++ b/include/bycorf/rdma/rc.h @@ -0,0 +1,111 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#if BYCORF_ENABLE_RDMA +#include + +#include +#include +#include +#include +#include +#include + +#include "bycorf/io/rdma_memory.h" +#include "bycorf/runtime/task.h" + +namespace bycorf { +class Worker; +namespace rdma { +namespace detail { +struct Connection; +struct Listener; +class Engine; +class RcTestPeer; +} // namespace detail + +// Fixed per-QP capacities; a shared CQ reserves the provider's actual send+recv +// capacities before admitting the QP. No CQ growth or receive-buffer +// allocation. +struct RcOptions { + std::uint32_t send_wr = 128; + std::uint32_t recv_wr = 128; + std::uint32_t cq_entries = 4096; + std::chrono::milliseconds connect_timeout{5000}; +}; + +// Owner-worker-only RC connection. Move-only; destruction requests asynchronous +// close. Await Close() to join resource retirement while the worker is running. +// Worker shutdown also closes all connections before reclaiming coroutine +// frames. +class RcConnection { + public: + RcConnection() = default; + RcConnection(RcConnection&&) noexcept; + RcConnection& operator=(RcConnection&&) noexcept; + RcConnection(const RcConnection&) = delete; + RcConnection& operator=(const RcConnection&) = delete; + ~RcConnection(); + + // Numeric IPv4/IPv6 only. Copies arguments before returning the lazy task. + // The deadline covers address/route resolution and the RC handshake. Private + // data is limited to 56 bytes for portable RC CM requests and responses. + static Task> Connect( + Worker& worker, std::string_view ip, std::uint16_t port, + RcOptions options = {}, std::span private_data = {}); + // Idempotent; one Close waiter at a time. Completion joins this connection's + // timer/event references and QP/CM ID destruction, not merely cancellation. + // Shared CM/device fd polls belong to the worker, not individual connections. + Task Close(); + bool IsOpen() const noexcept; + // Preserves the specific CM/provider failure; local close reports Cancelled. + absl::Status status() const; + std::span peer_private_data() const noexcept; + // Retain this handle through registered memory use; it is the QP's actual PD. + RdmaProtectionDomain protection_domain() const; + + private: + friend class RcListener; + friend class detail::RcTestPeer; + explicit RcConnection(std::shared_ptr state); + static Task> ConnectImpl( + std::shared_ptr engine, sockaddr_storage address, + RcOptions options, std::vector data); + std::shared_ptr state_; +}; + +// A listener and every accepted connection stay on the listening worker. +// Accepted connections already returned to the application outlive the +// listener. +class RcListener { + public: + RcListener() = default; + RcListener(RcListener&&) noexcept; + RcListener& operator=(RcListener&&) noexcept; + RcListener(const RcListener&) = delete; + RcListener& operator=(const RcListener&) = delete; + ~RcListener(); + + // Binds a numeric local address; port zero selects an ephemeral port. Options + // and response private data are copied. Requests beyond backlog are rejected. + static absl::StatusOr Listen( + Worker& worker, std::string_view ip, std::uint16_t port, + RcOptions options = {}, std::span private_data = {}, + int backlog = 128); + // One waiter at a time. Returns after establishment or a specific request's + // failure; queued and handshaking children are closed by listener Close(). + Task> Accept(); + Task Close(); + std::uint16_t port() const noexcept; + + private: + friend class detail::RcTestPeer; + static Task> AcceptImpl( + std::shared_ptr state); + std::shared_ptr state_; +}; +} // namespace rdma +} // namespace bycorf +#endif diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index 3fe2512..1ca3514 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -91,6 +91,7 @@ class Worker : private IoUringBackend, using Rdma::CancelRdmaEventPoll; using Rdma::OpenRdmaDevice; using Rdma::RegisterRdmaMemory; + using Rdma::SubmitRdmaDeadline; using Rdma::SubmitRdmaEventPoll; #endif struct WakeStats { @@ -386,6 +387,8 @@ class Worker : private IoUringBackend, friend class StorageBackend; #if BYCORF_ENABLE_RDMA friend class RdmaBackend; + friend class rdma::RcConnection; + friend class rdma::RcListener; #endif // Shared-ring access for the CRTP facades; the base conversion is a fixed // address adjustment rather than a load of a stored pointer. diff --git a/src/io/io_uring_backend.cpp b/src/io/io_uring_backend.cpp index 65ff574..60cd0ef 100644 --- a/src/io/io_uring_backend.cpp +++ b/src/io/io_uring_backend.cpp @@ -374,6 +374,15 @@ absl::Status IoUringBackend::SubmitTimeout(const __kernel_timespec& timeout, }); } +absl::Status IoUringBackend::SubmitDeadline(const __kernel_timespec& deadline, + IoCompletion* tag) { + auto* deadline_ptr = const_cast<__kernel_timespec*>(&deadline); + return SubmitOrQueue([deadline_ptr, tag](io_uring_sqe* sqe) { + io_uring_prep_timeout(sqe, deadline_ptr, 0, IORING_TIMEOUT_ABS); + io_uring_sqe_set_data(sqe, tag); + }); +} + io_uring_sqe* IoUringBackend::AcquireSqe() { if (!initialized_) { return nullptr; diff --git a/src/rdma/rc.cpp b/src/rdma/rc.cpp new file mode 100644 index 0000000..9316bc2 --- /dev/null +++ b/src/rdma/rc.cpp @@ -0,0 +1,199 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "bycorf/rdma/rc.h" + +#include + +#include +#include +#include + +#include "bycorf/runtime/worker.h" +#include "rc_internal.h" + +namespace bycorf::rdma { +namespace { +absl::Status Validate(const RcOptions& options, + std::span data) { + if (options.send_wr == 0 || options.recv_wr == 0 || options.cq_entries == 0 || + options.cq_entries > + static_cast(std::numeric_limits::max()) || + options.connect_timeout.count() <= 0 || + options.connect_timeout.count() > std::numeric_limits::max() || + data.size() > 56) + return absl::InvalidArgumentError( + "invalid RDMA capacity, timeout or private data length"); + return absl::OkStatus(); +} +absl::StatusOr Address(std::string_view ip, + std::uint16_t port) { + sockaddr_storage address{}; + std::string text(ip); + auto* v4 = reinterpret_cast(&address); + if (inet_pton(AF_INET, text.c_str(), &v4->sin_addr) == 1) { + v4->sin_family = AF_INET; + v4->sin_port = htons(port); + return address; + } + auto* v6 = reinterpret_cast(&address); + if (inet_pton(AF_INET6, text.c_str(), &v6->sin6_addr) == 1) { + v6->sin6_family = AF_INET6; + v6->sin6_port = htons(port); + return address; + } + return absl::InvalidArgumentError( + "RDMA requires a numeric IPv4 or IPv6 address"); +} +Task> FailedConnect(absl::Status error) { + co_return error; +} + +template +void Drop(const std::shared_ptr& state) { + if (!state || state->closed) return; + if (auto engine = state->engine.lock()) { + // Destruction can request asynchronous close but cannot migrate ownership. + // After worker shutdown all public states are already retired. + if (!engine->OnOwner() && !engine->stopping) std::terminate(); + engine->Close(state, absl::CancelledError("RDMA handle closed")); + } +} +template +Task CloseImpl(std::shared_ptr state) { + if (!state || state->closed) co_return absl::OkStatus(); + auto engine = state->engine.lock(); + if (!engine) co_return absl::CancelledError("RDMA owner is gone"); + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError("RDMA close on a different worker"); + engine->Close(state, absl::CancelledError("RDMA handle closed")); + co_return co_await detail::Wait{state->retired}; +} +struct ConnectGuard { + std::shared_ptr state; + bool delivered = false; + ~ConnectGuard() { + if (!delivered) Drop(state); + } +}; +} // namespace + +RcConnection::RcConnection(std::shared_ptr state) + : state_(std::move(state)) {} +RcConnection::RcConnection(RcConnection&&) noexcept = default; +RcConnection& RcConnection::operator=(RcConnection&& other) noexcept { + if (this != &other) { + Drop(state_); + state_ = std::move(other.state_); + } + return *this; +} +RcConnection::~RcConnection() { Drop(state_); } +Task> RcConnection::Connect( + Worker& worker, std::string_view ip, std::uint16_t port, RcOptions options, + std::span private_data) { + auto status = Validate(options, private_data); + if (!status.ok()) return FailedConnect(status); + if (port == 0) + return FailedConnect( + absl::InvalidArgumentError("RDMA connect port is zero")); + auto address = Address(ip, port); + if (!address.ok()) return FailedConnect(address.status()); + auto engine = worker.GetRdmaEngine(); + if (!engine.ok()) return FailedConnect(engine.status()); + return ConnectImpl(*engine, *address, options, + {private_data.begin(), private_data.end()}); +} +Task> RcConnection::ConnectImpl( + std::shared_ptr engine, sockaddr_storage address, + RcOptions options, std::vector data) { + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError( + "RDMA connect on a different worker"); + auto started = engine->Connect(address, options, std::move(data)); + if (!started.ok()) co_return started.status(); + auto c = *started; + ConnectGuard guard{c}; + auto status = co_await detail::Wait{c->connected}; + if (!status.ok()) co_return status; + if (!c->error.ok()) co_return c->error; + guard.delivered = true; + co_return RcConnection(std::move(c)); +} +Task RcConnection::Close() { return CloseImpl(state_); } +bool RcConnection::IsOpen() const noexcept { + return state_ && state_->established && !state_->closing; +} +absl::Status RcConnection::status() const { + return state_ ? state_->error + : absl::FailedPreconditionError("empty RDMA connection"); +} +std::span RcConnection::peer_private_data() const noexcept { + return state_ ? std::span(state_->peer_data) + : std::span{}; +} +RdmaProtectionDomain RcConnection::protection_domain() const { + return state_ && state_->device ? state_->device->domain + : RdmaProtectionDomain{}; +} + +RcListener::RcListener(RcListener&&) noexcept = default; +RcListener& RcListener::operator=(RcListener&& other) noexcept { + if (this != &other) { + Drop(state_); + state_ = std::move(other.state_); + } + return *this; +} +RcListener::~RcListener() { Drop(state_); } +absl::StatusOr RcListener::Listen( + Worker& worker, std::string_view ip, std::uint16_t port, RcOptions options, + std::span private_data, int backlog) { + auto status = Validate(options, private_data); + if (!status.ok()) return status; + if (backlog <= 0) + return absl::InvalidArgumentError("RDMA backlog must be positive"); + auto address = Address(ip, port); + if (!address.ok()) return address.status(); + auto engine = worker.GetRdmaEngine(); + if (!engine.ok()) return engine.status(); + auto started = (*engine)->Listen( + *address, options, {private_data.begin(), private_data.end()}, backlog); + if (!started.ok()) return started.status(); + RcListener listener; + listener.state_ = *started; + return listener; +} +Task> RcListener::Accept() { + return AcceptImpl(state_); +} +Task> RcListener::AcceptImpl( + std::shared_ptr state) { + if (!state) co_return absl::FailedPreconditionError("empty RDMA listener"); + auto engine = state->engine.lock(); + if (!engine || state->closing) + co_return state->error.ok() ? absl::CancelledError("RDMA listener closed") + : state->error; + if (!engine->OnOwner()) + co_return absl::FailedPreconditionError( + "RDMA accept on a different worker"); + if (state->accepted.waiter) + co_return absl::FailedPreconditionError("concurrent RDMA accept"); + while (state->ready.empty() && !state->closing) { + state->accepted.ready = false; + auto status = co_await detail::Wait{state->accepted}; + if (!status.ok()) co_return status; + } + if (state->closing) co_return state->error; + auto c = std::move(state->ready.front()); + state->ready.pop_front(); + state->children.erase(c.get()); + c->listener.reset(); + if (!c->error.ok()) co_return c->error; + co_return RcConnection(std::move(c)); +} +Task RcListener::Close() { return CloseImpl(state_); } +std::uint16_t RcListener::port() const noexcept { + return state_ ? state_->port : 0; +} +} // namespace bycorf::rdma diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp new file mode 100644 index 0000000..3766d50 --- /dev/null +++ b/src/rdma/rc_engine.cpp @@ -0,0 +1,761 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "bycorf/io/rdma_reactor.h" +#include "bycorf/runtime/worker.h" +#include "rc_internal.h" +#include "spdlog/spdlog.h" + +namespace bycorf::rdma::detail { +namespace { +absl::Status Nonblocking(int fd) { + int flags = fcntl(fd, F_GETFL); + if (flags < 0 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) + return ErrnoStatus("RDMA channel nonblocking setup"); + return absl::OkStatus(); +} +bool Again() { return errno == EAGAIN || errno == EWOULDBLOCK; } + +template +auto Snapshot(const Map& map) { + std::vector result; + result.reserve(map.size()); + for (const auto& [key, value] : map) result.push_back(value); + return result; +} + +absl::Status CmError(rdma_cm_event_type event, int status) { + // Rejection status can be a transport reason, not errno. Preserve both the + // event and raw status rather than accidentally interpreting it as errno. + return absl::UnavailableError( + absl::StrCat(rdma_event_str(event), " (status ", status, ")")); +} +rdma_conn_param Parameters(const Connection& c) { + rdma_conn_param p{}; + p.private_data = c.local_data.empty() ? nullptr : c.local_data.data(); + p.private_data_len = static_cast(c.local_data.size()); + p.responder_resources = c.responder_resources; + p.initiator_depth = c.initiator_depth; + p.retry_count = 7; + p.rnr_retry_count = 7; + p.qp_num = c.qp->qp_num; + return p; +} +} // namespace + +absl::Status ErrnoStatus(const char* operation) { + int error = errno; + return absl::ErrnoToStatus(error ? error : EIO, + absl::StrCat(operation, " (errno ", error, ")")); +} +void CheckDestroy(int result, const char* operation) { + if (result != 0) { + spdlog::critical("{} failed while retiring RDMA resources: {}", operation, + result); + std::terminate(); + } +} +void Signal::Publish(Worker* worker) { + if (ready) return; + ready = true; + if (worker && waiter) worker->Enqueue(waiter); +} +Wait::~Wait() { + if (signal.waiter == handle) signal.waiter = {}; +} +bool Wait::await_suspend(std::coroutine_handle<> awaiting) { + if (signal.waiter) { + conflict = true; + return false; + } + handle = awaiting; + signal.waiter = awaiting; + return true; +} +absl::Status Wait::await_resume() { + return conflict ? absl::FailedPreconditionError("concurrent RDMA wait") + : absl::OkStatus(); +} + +absl::Status Poll::Arm() { + if (armed || stopping) return absl::OkStatus(); + auto status = engine->worker->SubmitRdmaEventPoll(fd, POLLIN, this); + if (status.ok()) armed = true; + return status; +} +void Poll::Stop() { + if (stopping) return; + stopping = true; + if (armed) { + // Cancellation admission is not retirement. Keep the tag, descriptor and + // resource owner until Complete, or the explicit post-ring-shutdown phase. + const auto status = engine->worker->CancelRdmaEventPoll(this); + if (!status.ok()) engine->worker->RequestStop(); + } +} +void Poll::Complete(Worker&, int result, unsigned) { + // Failure fanout may remove the device cache entry. This event reference + // keeps the containing poll alive until the callback has fully returned. + auto retained = device ? device->shared_from_this() : nullptr; + armed = false; + if (!stopping) engine->OnPoll(*this, result); + // A cancellation CQE is the retirement boundary; cancellation admission + // alone never permits release of the descriptor or containing Device. + if (device) engine->RetireFailedDevice(*device); +} +void Deadline::Complete(Worker&, int result, unsigned) { + auto it = engine->connections.find(connection->id); + if (it == engine->connections.end()) std::terminate(); + auto retained = it->second; + armed = false; + if (!retained->handshake_done) { + engine->Close(retained, result == -ETIME + ? absl::DeadlineExceededError( + "RDMA connection deadline exceeded") + : absl::UnknownError(absl::StrCat( + "RDMA deadline completion: ", result))); + } + engine->Progress(retained); +} + +Device::~Device() { + if (cq) CheckDestroy(ibv_destroy_cq(cq), "ibv_destroy_cq"); + if (channel) + CheckDestroy(ibv_destroy_comp_channel(channel), "ibv_destroy_comp_channel"); +} +Engine::~Engine() { + // Normal destruction follows AfterRingShutdown. Failed Init has no accepted + // poll, and can release its channel directly. + if (cm) rdma_destroy_event_channel(cm); +} +bool Engine::OnOwner() const { return worker && ThisWorker().self_ == worker; } +absl::Status Engine::Init() { + cm = rdma_create_event_channel(); + if (!cm) return ErrnoStatus("rdma_create_event_channel"); + auto status = Nonblocking(cm->fd); + if (!status.ok()) return status; + cm_poll.engine = this; + cm_poll.fd = cm->fd; + return cm_poll.Arm(); +} +absl::StatusOr> CreateEngine(Worker& worker) { + if (ThisWorker().self_ != &worker) + return absl::FailedPreconditionError("RDMA must run on its owner worker"); + auto engine = std::make_shared(worker); + auto status = engine->Init(); + if (!status.ok()) return status; + return engine; +} +void BeginShutdown(const std::shared_ptr& engine) { + if (engine) engine->BeginShutdown(); +} +void AfterRingShutdown(const std::shared_ptr& engine) { + if (engine) engine->AfterRingShutdown(); +} + +absl::StatusOr> Engine::Listen( + const sockaddr_storage& address, RcOptions options, + std::vector data, int backlog) { + if (stopping) return absl::CancelledError("RDMA worker is shutting down"); + if (!OnOwner()) + return absl::FailedPreconditionError("RDMA call on a different worker"); + if (!error.ok()) return error; + auto listener = std::make_shared(); + listener->engine = shared_from_this(); + listener->options = options; + listener->private_data = std::move(data); + listener->backlog = backlog; + if (rdma_create_id(cm, &listener->id, nullptr, RDMA_PS_TCP)) + return ErrnoStatus("rdma_create_id(listener)"); + auto addr = address; + if (rdma_bind_addr(listener->id, reinterpret_cast(&addr)) || + rdma_listen(listener->id, backlog)) { + auto status = ErrnoStatus("RDMA bind/listen"); + CheckDestroy(rdma_destroy_id(listener->id), "rdma_destroy_id(listener)"); + listener->id = nullptr; + return status; + } + listener->port = ntohs(rdma_get_src_port(listener->id)); + listeners.emplace(listener->id, listener); + return listener; +} +absl::Status Engine::StartTimer(const std::shared_ptr& c) { + c->expires = std::chrono::steady_clock::now() + c->options.connect_timeout; + c->timer.engine = this; + c->timer.connection = c.get(); + const auto ms = c->options.connect_timeout.count(); + timespec now{}; + if (clock_gettime(CLOCK_MONOTONIC, &now)) return ErrnoStatus("clock_gettime"); + const auto nanos = now.tv_nsec + (ms % 1000) * 1000000; + c->timer.expires = {.tv_sec = now.tv_sec + ms / 1000 + nanos / 1000000000, + .tv_nsec = nanos % 1000000000}; + auto status = worker->SubmitRdmaDeadline(c->timer.expires, &c->timer); + if (status.ok()) c->timer.armed = true; + return status; +} +absl::StatusOr> Engine::Connect( + const sockaddr_storage& address, RcOptions options, + std::vector data) { + if (stopping) return absl::CancelledError("RDMA worker is shutting down"); + if (!OnOwner()) + return absl::FailedPreconditionError("RDMA call on a different worker"); + if (!error.ok()) return error; + auto c = std::make_shared(); + c->engine = shared_from_this(); + c->options = options; + c->local_data = std::move(data); + if (rdma_create_id(cm, &c->id, nullptr, RDMA_PS_TCP)) + return ErrnoStatus("rdma_create_id(connect)"); + connections.emplace(c->id, c); + auto status = StartTimer(c); + auto addr = address; + if (status.ok() && + rdma_resolve_addr(c->id, nullptr, reinterpret_cast(&addr), + static_cast(options.connect_timeout.count()))) + status = ErrnoStatus("rdma_resolve_addr"); + if (!status.ok()) Close(c, status); + return c; +} + +absl::StatusOr> Engine::GetDevice(Connection& c) { + auto* selected = c.id->verbs; + if (!selected || !selected->device) + return absl::FailedPreconditionError("CM did not select a verbs device"); + const std::string name = ibv_get_device_name(selected->device); + if (auto it = devices.find(name); it != devices.end()) { + if (!it->second->error.ok()) return it->second->error; + if (it->second->requested_cqe != c.options.cq_entries) + return absl::InvalidArgumentError( + "shared RDMA CQ capacity is already fixed"); + return it->second; + } + auto device = std::make_shared(); + auto* opened = ibv_open_device(selected->device); + if (!opened) return ErrnoStatus("ibv_open_device(RC)"); + device->context = + std::shared_ptr(opened, [](ibv_context* context) { + CheckDestroy(ibv_close_device(context), "ibv_close_device(RC)"); + }); + auto domain = worker->AcquireRdmaDomain(device->context); + if (!domain.ok()) return domain.status(); + device->domain = *domain; + ibv_device_attr attrs{}; + int result = ibv_query_device(opened, &attrs); + if (result) return absl::ErrnoToStatus(result, "ibv_query_device"); + if (c.options.cq_entries > static_cast(attrs.max_cqe)) + return absl::ResourceExhaustedError("requested CQ exceeds device max_cqe"); + device->channel = ibv_create_comp_channel(opened); + if (!device->channel) return ErrnoStatus("ibv_create_comp_channel"); + auto status = Nonblocking(device->channel->fd); + if (!status.ok()) return status; + status = Nonblocking(opened->async_fd); + if (!status.ok()) return status; + device->cq = ibv_create_cq(opened, static_cast(c.options.cq_entries), + device.get(), device->channel, 0); + if (!device->cq) return ErrnoStatus("ibv_create_cq"); + device->requested_cqe = c.options.cq_entries; + device->completion_poll.engine = device->async_poll.engine = this; + device->completion_poll.device = device->async_poll.device = device.get(); + device->completion_poll.kind = Poll::Kind::kCq; + device->completion_poll.fd = device->channel->fd; + device->async_poll.kind = Poll::Kind::kAsync; + device->async_poll.fd = opened->async_fd; + // Retain the device BEFORE admitting any fd poll. A later failure must not + // destroy a tag or fd that the ring may still reference. + devices.emplace(name, device); + result = ibv_req_notify_cq(device->cq, 0); + if (result) status = absl::ErrnoToStatus(result, "ibv_req_notify_cq"); + if (status.ok()) DrainCq(*device); // arm then recheck, including startup + if (status.ok()) status = device->error; + if (status.ok()) status = device->completion_poll.Arm(); + if (status.ok()) status = device->async_poll.Arm(); + if (!status.ok()) { + FailDevice(*device, status); + RetireFailedDevice(*device); + return status; + } + return device; +} +absl::Status Engine::Transition(Connection& c, ibv_qp_state state) { + ibv_qp_attr attr{}; + attr.qp_state = state; + int mask = 0; + if (rdma_init_qp_attr(c.id, &attr, &mask)) + return ErrnoStatus("rdma_init_qp_attr"); + if (state == IBV_QPS_RTR) attr.max_dest_rd_atomic = c.responder_resources; + if (state == IBV_QPS_RTS) attr.max_rd_atomic = c.initiator_depth; + const int result = ibv_modify_qp(c.qp, &attr, mask); + return result ? absl::ErrnoToStatus(result, "ibv_modify_qp(RC)") + : absl::OkStatus(); +} +absl::Status Engine::CreateQp(const std::shared_ptr& c) { + auto found = GetDevice(*c); + if (!found.ok()) return found.status(); + auto device = *found; + const std::uint64_t requested = + std::uint64_t(c->options.send_wr) + c->options.recv_wr; + if (requested > + static_cast(device->cq->cqe) - device->reserved) + return absl::ResourceExhaustedError("shared RDMA CQ budget exhausted"); + ibv_qp_init_attr attr{}; + attr.qp_type = IBV_QPT_RC; + attr.send_cq = attr.recv_cq = device->cq; + attr.cap.max_send_wr = c->options.send_wr; + attr.cap.max_recv_wr = c->options.recv_wr; + attr.cap.max_send_sge = attr.cap.max_recv_sge = 1; + attr.sq_sig_all = 1; + c->qp = ibv_create_qp(device->domain.native_pd(), &attr); + if (!c->qp) return ErrnoStatus("ibv_create_qp(RC)"); + // Providers may round capacities up. Until the actual budget is reserved, + // this QP remains unconnected with no work posted, so it cannot fill the CQ. + const std::uint64_t actual = + std::uint64_t(attr.cap.max_send_wr) + attr.cap.max_recv_wr; + if (actual > static_cast(device->cq->cqe) - device->reserved) { + CheckDestroy(ibv_destroy_qp(c->qp), "ibv_destroy_qp(over budget)"); + c->qp = nullptr; + return absl::ResourceExhaustedError( + "provider QP capacities exceed shared CQ budget"); + } + c->device = device; + c->budget = actual; + device->reserved += actual; + // External-QP CM mode: never assign id->qp. CM routes/negotiates, while the + // worker's own context owns QP/PD/MR/CQ and the sole async-event reader. + return Transition(*c, IBV_QPS_INIT); +} + +void Engine::Established(const std::shared_ptr& c) { + if (std::chrono::steady_clock::now() >= c->expires) { + Close(c, absl::DeadlineExceededError("RDMA connection deadline exceeded")); + return; + } + c->established = true; + c->handshake_done = true; + if (c->timer.armed) { + auto status = worker->CancelRdmaEventPoll(&c->timer); + if (!status.ok()) { + Close(c, status); + return; + } + } + Progress(c); +} +void Engine::HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, + rdma_cm_event_type event, int status, + std::vector data, rdma_conn_param peer) { + if (event == RDMA_CM_EVENT_DEVICE_REMOVAL) { + // Retain endpoints before fanout: failing the device can erase their map + // entries and destroy the CM ID. Nonzero status still means device scope. + auto ci = connections.find(id); + auto li = listeners.find(id); + auto connection = ci == connections.end() ? nullptr : ci->second; + auto listener = li == listeners.end() ? nullptr : li->second; + const auto error = CmError(event, status); + if (id->verbs && id->verbs->device) { + auto found = devices.find(ibv_get_device_name(id->verbs->device)); + if (found != devices.end()) FailDevice(*found->second, error); + } + if (connection) Close(connection, error); + if (listener) Close(listener, error); + return; + } + if (event == RDMA_CM_EVENT_CONNECT_REQUEST) { + auto it = listeners.find(listen_id); + if (stopping || it == listeners.end() || it->second->closing || + it->second->children.size() >= + static_cast(it->second->backlog)) { + (void)rdma_reject(id, nullptr, 0); + CheckDestroy(rdma_destroy_id(id), "rdma_destroy_id(rejected request)"); + return; + } + auto listener = it->second; + auto c = std::make_shared(); + c->engine = shared_from_this(); + c->listener = listener; + c->id = id; + c->options = listener->options; + c->local_data = listener->private_data; + c->peer_data = std::move(data); + c->initiator_depth = std::min(1, peer.responder_resources); + c->responder_resources = std::min(1, peer.initiator_depth); + connections.emplace(id, c); + listener->children.emplace(c.get(), c); + auto result = StartTimer(c); + if (result.ok()) result = CreateQp(c); + if (result.ok()) result = Transition(*c, IBV_QPS_RTR); + if (result.ok()) result = Transition(*c, IBV_QPS_RTS); + if (result.ok()) { + auto param = Parameters(*c); + if (rdma_accept(c->id, ¶m)) + result = ErrnoStatus("rdma_accept"); + else + c->cm_started = true; + } + if (!result.ok()) { + (void)rdma_reject(id, nullptr, 0); + Close(c, result); + } + return; + } + auto it = connections.find(id); + if (it == connections.end()) { + if (auto listener = listeners.find(id); listener != listeners.end()) + Close(listener->second, CmError(event, status)); + return; + } + auto c = it->second; + if (c->closing) return; + if (status || event == RDMA_CM_EVENT_REJECTED || + event == RDMA_CM_EVENT_CONNECT_ERROR || + event == RDMA_CM_EVENT_ADDR_ERROR || event == RDMA_CM_EVENT_ROUTE_ERROR || + event == RDMA_CM_EVENT_UNREACHABLE || + event == RDMA_CM_EVENT_DISCONNECTED) { + if (!data.empty()) c->peer_data = std::move(data); + Close(c, CmError(event, status)); + return; + } + absl::Status result; + switch (event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + if (rdma_resolve_route( + id, static_cast(c->options.connect_timeout.count()))) + result = ErrnoStatus("rdma_resolve_route"); + break; + case RDMA_CM_EVENT_ROUTE_RESOLVED: + result = CreateQp(c); + if (result.ok()) { + auto param = Parameters(*c); + if (rdma_connect(id, ¶m)) + result = ErrnoStatus("rdma_connect"); + else + c->cm_started = true; + } + break; + case RDMA_CM_EVENT_CONNECT_RESPONSE: + c->peer_data = std::move(data); + c->initiator_depth = + std::min(c->initiator_depth, peer.responder_resources); + c->responder_resources = + std::min(c->responder_resources, peer.initiator_depth); + result = Transition(*c, IBV_QPS_INIT); + if (result.ok()) result = Transition(*c, IBV_QPS_RTR); + if (result.ok()) result = Transition(*c, IBV_QPS_RTS); + if (result.ok() && rdma_establish(id)) + result = ErrnoStatus("rdma_establish"); + if (result.ok()) Established(c); + break; + case RDMA_CM_EVENT_ESTABLISHED: + Established(c); + break; + case RDMA_CM_EVENT_DEVICE_REMOVAL: + if (c->device) + FailDevice(*c->device, CmError(event, status)); + else + Close(c, CmError(event, status)); + break; + case RDMA_CM_EVENT_TIMEWAIT_EXIT: + break; + default: + result = CmError(event, status); + break; + } + if (!result.ok()) Close(c, result); +} +void Engine::DrainCm() { + for (;;) { + rdma_cm_event* event = nullptr; + if (rdma_get_cm_event(cm, &event)) { + if (errno == EINTR) continue; + if (!Again()) FailAll(ErrnoStatus("rdma_get_cm_event")); + return; + } + auto* id = event->id; + auto* listener = event->listen_id; + const auto kind = event->event; + const int status = event->status; + std::vector data; + rdma_conn_param peer{}; + if (kind == RDMA_CM_EVENT_CONNECT_REQUEST || + kind == RDMA_CM_EVENT_CONNECT_RESPONSE || + kind == RDMA_CM_EVENT_ESTABLISHED || kind == RDMA_CM_EVENT_REJECTED) { + peer = event->param.conn; + peer.private_data = nullptr; + const auto* first = + static_cast(event->param.conn.private_data); + if (first && event->param.conn.private_data_len) + data.assign(first, first + event->param.conn.private_data_len); + } + // Copy before ack, then dispatch. No handler may destroy an ID while an + // unacknowledged CM event still refers to it (rdma_destroy_id would wait). + CheckDestroy(rdma_ack_cm_event(event), "rdma_ack_cm_event"); + HandleCm(id, listener, kind, status, std::move(data), peer); + } +} + +void Engine::DrainCq(Device& device) { + ibv_wc completions[32]; + for (;;) { + const int count = ibv_poll_cq(device.cq, 32, completions); + if (count < 0) { + FailDevice(device, absl::UnavailableError("ibv_poll_cq failed")); + return; + } + if (!count) return; + for (int i = 0; i < count; ++i) { + const auto& wc = completions[i]; + // No public data-posting API exists yet. Error completions still + // terminate their connection; success entries are consumed so + // notifications rearm. + if (wc.status == IBV_WC_SUCCESS) continue; + for (auto& c : Snapshot(connections)) { + if (c->device.get() == &device && c->qp && c->qp->qp_num == wc.qp_num) { + Close(c, absl::UnavailableError(absl::StrCat( + "RDMA completion: ", ibv_wc_status_str(wc.status), + " (vendor ", wc.vendor_err, ")"))); + } + } + } + } +} +void Engine::DrainNotifications(Device& device, bool rearm) { + for (;;) { + ibv_cq* cq = nullptr; + void* context = nullptr; + if (ibv_get_cq_event(device.channel, &cq, &context)) { + if (errno == EINTR) continue; + if (!Again() && rearm) + FailDevice(device, ErrnoStatus("ibv_get_cq_event")); + return; + } + ibv_ack_cq_events(cq, 1); + if (cq != device.cq) { + FailDevice(device, absl::InternalError( + "unexpected CQ on worker completion channel")); + continue; + } + if (rearm && device.error.ok()) { + const int result = ibv_req_notify_cq(cq, 0); + if (result) { + FailDevice(device, absl::ErrnoToStatus(result, "ibv_req_notify_cq")); + continue; + } + // Notification is one-shot. Rearm BEFORE draining: a completion racing + // this drain is either consumed here or leaves a channel event to wake + // us. + DrainCq(device); + } + } +} +void Engine::DrainAsync(Device& device) { + for (;;) { + ibv_async_event event{}; + if (ibv_get_async_event(device.context.get(), &event)) { + if (errno == EINTR) continue; + if (!Again()) FailDevice(device, ErrnoStatus("ibv_get_async_event")); + return; + } + const auto type = event.event_type; + const auto element = event.element; + ibv_ack_async_event(&event); + auto error = absl::UnavailableError( + absl::StrCat("RDMA device event: ", ibv_event_type_str(type))); + if (type == IBV_EVENT_DEVICE_FATAL || type == IBV_EVENT_CQ_ERR) { + FailDevice(device, error); + } else if (type == IBV_EVENT_QP_FATAL || type == IBV_EVENT_QP_REQ_ERR || + type == IBV_EVENT_QP_ACCESS_ERR || type == IBV_EVENT_PORT_ERR) { + for (auto& c : Snapshot(connections)) { + if (c->device.get() == &device && + ((type == IBV_EVENT_PORT_ERR && + c->id->port_num == element.port_num) || + (type != IBV_EVENT_PORT_ERR && c->qp == element.qp))) + Close(c, error); + } + } + } +} +void Engine::OnPoll(Poll& poll, int result) { + if (result < 0 || (result & (POLLERR | POLLHUP | POLLNVAL))) { + auto error = + absl::UnavailableError(absl::StrCat("RDMA fd poll failed: ", result)); + if (poll.device) + FailDevice(*poll.device, error); + else + FailAll(error); + return; + } + switch (poll.kind) { + case Poll::Kind::kCm: + DrainCm(); + break; + case Poll::Kind::kCq: + DrainNotifications(*poll.device, true); + break; + case Poll::Kind::kAsync: + DrainAsync(*poll.device); + break; + } + auto status = poll.Arm(); + if (!status.ok()) { + if (poll.device) + FailDevice(*poll.device, status); + else + FailAll(status); + } +} +void Engine::FailDevice(Device& device, const absl::Status& error) { + if (!device.error.ok()) return; + device.error = error; + device.completion_poll.Stop(); + device.async_poll.Stop(); + for (auto& c : Snapshot(connections)) + if (c->device.get() == &device) Close(c, error); +} +void Engine::RetireFailedDevice(Device& device) { + if (stopping || device.error.ok() || device.reserved != 0 || + device.completion_poll.armed || device.async_poll.armed) + return; + DrainNotifications(device, false); + DrainAsync(device); + for (auto it = devices.begin(); it != devices.end(); ++it) { + if (it->second.get() == &device) { + devices.erase(it); + return; + } + } +} +void Engine::FailAll(const absl::Status& error) { + if (this->error.ok()) this->error = error; + cm_poll.Stop(); + for (auto& listener : Snapshot(listeners)) Close(listener, error); + for (auto& c : Snapshot(connections)) Close(c, error); +} + +void Engine::Close(std::shared_ptr c, absl::Status reason) { + if (c->closing || c->closed) return; + c->closing = true; + c->handshake_done = true; + if (c->error.ok()) c->error = std::move(reason); + if (c->cm_started) (void)rdma_disconnect(c->id); + // External QPs are not transitioned by rdma_disconnect. Posting stops before + // ERR, and CQ capacity is not returned until the QP is destroyed and drained. + if (c->qp) { + ibv_qp_attr attr{}; + attr.qp_state = IBV_QPS_ERR; + (void)ibv_modify_qp(c->qp, &attr, IBV_QP_STATE); + } + if (c->timer.armed) { + auto status = worker->CancelRdmaEventPoll(&c->timer); + if (!status.ok()) worker->RequestStop(); + } + Progress(c); +} +void Engine::Progress(std::shared_ptr c) { + if (c->timer.armed || c->closed) return; + if (c->closing) { + // Erase before any CQ drain can fail another connection and reenter this + // path. The local shared reference retains the object through callbacks. + connections.erase(c->id); + if (c->qp) { + auto* qp = std::exchange(c->qp, nullptr); + CheckDestroy(ibv_destroy_qp(qp), "ibv_destroy_qp(RC)"); + DrainCq(*c->device); + c->device->reserved -= c->budget; + c->budget = 0; + RetireFailedDevice(*c->device); + } + CheckDestroy(rdma_destroy_id(c->id), "rdma_destroy_id(RC)"); + c->id = nullptr; + c->device.reset(); + c->closed = true; + c->retired.Publish(wake_worker()); + } + if (c->handshake_done) { + c->connected.Publish(wake_worker()); + if (auto listener = c->listener.lock()) { + if (!c->queued && !listener->closing) { + c->queued = true; + listener->ready.push_back(c); + listener->accepted.Publish(wake_worker()); + } + Progress(listener); + } + } +} +void Engine::Close(std::shared_ptr listener, absl::Status reason) { + if (listener->closing || listener->closed) return; + listener->closing = true; + listener->error = reason; + // No per-listener fd tag exists: the shared CM poll references only Engine. + // DrainCm has acknowledged its event before this ID can be destroyed. + CheckDestroy(rdma_destroy_id(listener->id), "rdma_destroy_id(listener)"); + listener->id = nullptr; + for (auto& c : Snapshot(listener->children)) Close(c, reason); + listener->accepted.Publish(wake_worker()); + Progress(listener); +} +void Engine::Progress(std::shared_ptr listener) { + if (!listener->closing || listener->closed) return; + for (const auto& [key, c] : listener->children) + if (!c->closed) return; + listener->ready.clear(); + listener->children.clear(); + for (auto it = listeners.begin(); it != listeners.end(); ++it) { + if (it->second == listener) { + listeners.erase(it); + break; + } + } + listener->closed = true; + listener->retired.Publish(wake_worker()); +} +void Engine::BeginShutdown() { + if (stopping) return; + stopping = true; + cm_poll.Stop(); + for (auto& [name, device] : devices) { + device->completion_poll.Stop(); + device->async_poll.Stop(); + } + const auto error = absl::CancelledError("RDMA worker shutdown"); + for (auto& listener : Snapshot(listeners)) Close(listener, error); + for (auto& c : Snapshot(connections)) Close(c, error); +} +void Engine::AfterRingShutdown() { + // io_uring_queue_exit has joined kernel access, including queued/unsubmitted + // tags which never got callbacks. Only now may retained fd owners disappear. + cm_poll.armed = false; + for (auto& [name, device] : devices) { + device->completion_poll.armed = device->async_poll.armed = false; + DrainAsync(*device); + } + if (cm) DrainCm(); + for (auto& c : Snapshot(connections)) { + c->timer.armed = false; + Progress(c); + } + for (auto& listener : Snapshot(listeners)) Progress(listener); + for (auto& [name, device] : devices) { + DrainNotifications(*device, false); + DrainAsync(*device); + // Public connection states can survive shutdown; release device references + // there too, but exported PD/MR handles independently retain their context. + } + devices.clear(); + if (cm) { + rdma_destroy_event_channel(cm); + cm = nullptr; + } + worker = nullptr; +} +} // namespace bycorf::rdma::detail diff --git a/src/rdma/rc_internal.h b/src/rdma/rc_internal.h new file mode 100644 index 0000000..16078d0 --- /dev/null +++ b/src/rdma/rc_internal.h @@ -0,0 +1,160 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "bycorf/io/completion.h" +#include "bycorf/rdma/rc.h" + +namespace bycorf::rdma::detail { +struct Device; +struct Signal { + bool ready = false; + std::coroutine_handle<> waiter{}; + void Publish(Worker* worker); +}; + +// The waiter is owned by its coroutine, not by an fd tag. Shutdown may destroy +// that coroutine only after the ring is quiesced; this guard detaches it. +struct Wait { + Signal& signal; + std::coroutine_handle<> handle{}; + bool conflict = false; + ~Wait(); + bool await_ready() const noexcept { return signal.ready; } + bool await_suspend(std::coroutine_handle<> awaiting); + absl::Status await_resume(); +}; + +struct Poll final : IoCompletion { + enum class Kind { kCm, kCq, kAsync }; + Engine* engine = nullptr; + Device* device = nullptr; + Kind kind = Kind::kCm; + int fd = -1; + bool armed = false; + bool stopping = false; + absl::Status Arm(); + void Stop(); + void Complete(Worker&, int result, unsigned flags) override; +}; + +struct Deadline final : IoCompletion { + Engine* engine = nullptr; + Connection* connection = nullptr; + bool armed = false; + __kernel_timespec expires{}; + void Complete(Worker&, int result, unsigned flags) override; +}; + +struct Connection { + std::weak_ptr engine; + std::weak_ptr listener; + rdma_cm_id* id = nullptr; + ibv_qp* qp = nullptr; + std::shared_ptr device; + std::uint64_t budget = 0; + RcOptions options; + std::vector local_data, peer_data; + Deadline timer; + std::chrono::steady_clock::time_point expires; + bool established = false; + bool cm_started = false; + std::uint8_t initiator_depth = 1; + std::uint8_t responder_resources = 1; + bool handshake_done = false; + bool closing = false; + bool queued = false; + bool closed = false; + absl::Status error; + Signal connected, retired; +}; + +struct Listener { + std::weak_ptr engine; + rdma_cm_id* id = nullptr; + RcOptions options; + std::vector private_data; + std::uint16_t port = 0; + int backlog = 0; + bool closing = false; + bool closed = false; + absl::Status error; + std::unordered_map> children; + std::deque> ready; + Signal accepted, retired; +}; + +struct Device : std::enable_shared_from_this { + std::shared_ptr context; + RdmaProtectionDomain domain; + ibv_comp_channel* channel = nullptr; + ibv_cq* cq = nullptr; + std::uint32_t requested_cqe = 0; + std::uint64_t reserved = 0; + absl::Status error; + Poll completion_poll, async_poll; + // CM has process-shared contexts. This context is opened privately on the + // same actual device so its async event stream has exactly one worker reader. + ~Device(); +}; + +class Engine : public std::enable_shared_from_this { + public: + explicit Engine(Worker& owner) : worker(&owner) {} + ~Engine(); + absl::Status Init(); + absl::StatusOr> Listen( + const sockaddr_storage& address, RcOptions options, + std::vector data, int backlog); + absl::StatusOr> Connect( + const sockaddr_storage& address, RcOptions options, + std::vector data); + void Close(std::shared_ptr connection, absl::Status reason); + void Close(std::shared_ptr listener, absl::Status reason); + void Progress(std::shared_ptr connection); + void Progress(std::shared_ptr listener); + void OnPoll(Poll& poll, int result); + void DrainCm(); + void DrainCq(Device& device); + void DrainAsync(Device& device); + void FailDevice(Device& device, const absl::Status& error); + void RetireFailedDevice(Device& device); + void BeginShutdown(); + void AfterRingShutdown(); + bool OnOwner() const; + Worker* wake_worker() const { return stopping ? nullptr : worker; } + + Worker* worker; + bool stopping = false; + absl::Status error; + rdma_event_channel* cm = nullptr; + Poll cm_poll; + std::unordered_map> connections; + std::unordered_map> listeners; + std::unordered_map> devices; + + private: + absl::Status StartTimer(const std::shared_ptr& connection); + absl::Status CreateQp(const std::shared_ptr& connection); + absl::Status Transition(Connection& connection, ibv_qp_state state); + absl::StatusOr> GetDevice(Connection& connection); + void HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, rdma_cm_event_type event, + int status, std::vector data, rdma_conn_param peer); + void Established(const std::shared_ptr& connection); + void FailAll(const absl::Status& error); + void DrainNotifications(Device& device, bool rearm); +}; + +absl::Status ErrnoStatus(const char* operation); +void CheckDestroy(int result, const char* operation); +} // namespace bycorf::rdma::detail diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index 5a82d84..76cb8a1 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -237,17 +237,19 @@ absl::Status Worker::Init(const WorkerOptions& options) { void Worker::Shutdown() { #if BYCORF_ENABLE_RDMA - // Registration can be used before Init; its handles retain verbs resources - // independently of the ring. Never invalidate a live MR during shutdown. + // Stop RDMA admission first but retain fd tags and device resources until + // kernel shutdown confirms that no poll or timeout can reference them. Rdma::Shutdown(); #endif - if (!initialized_) { - return; + if (initialized_) { + Storage::Shutdown(); + Network::Shutdown(); + KernelBackend().Shutdown(); + initialized_ = false; } - Storage::Shutdown(); - Network::Shutdown(); - KernelBackend().Shutdown(); - initialized_ = false; +#if BYCORF_ENABLE_RDMA + Rdma::AfterRingShutdown(); +#endif } void Worker::Spawn(Task task) { diff --git a/tests/io_uring_backpressure_test.cpp b/tests/io_uring_backpressure_test.cpp index eacf08a..0955d30 100644 --- a/tests/io_uring_backpressure_test.cpp +++ b/tests/io_uring_backpressure_test.cpp @@ -19,8 +19,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -68,6 +70,25 @@ void Drain(Backend& backend, Predicate done) { } } +TEST(IoUringBackpressure, AbsoluteDeadlinesSurviveDeferredSubmission) { + // These objects outlive the backend, including on an assertion failure. + std::array tags; + timespec now{}; + ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &now), 0); + const __kernel_timespec expired{.tv_sec = now.tv_sec, .tv_nsec = now.tv_nsec}; + Fixture f; + ASSERT_NO_FATAL_FAILURE(f.Init()); + for (auto& tag : tags) + ASSERT_TRUE(f.backend.SubmitDeadline(expired, &tag).ok()); + // Sixteen requests exceed this fixture's two-entry SQ. Every request must + // expire after submission rather than restarting a relative timeout. + Drain(f.backend, [&] { + return std::all_of(tags.begin(), tags.end(), + [](const auto& tag) { return tag.calls == 1; }); + }); + for (const auto& tag : tags) EXPECT_EQ(tag.result, -ETIME); +} + TEST(IoUringBackpressure, SendsAndWake) { Fixture source, peer; ASSERT_NO_FATAL_FAILURE(source.Init()); diff --git a/tests/rdma_fake.cpp b/tests/rdma_fake.cpp new file mode 100644 index 0000000..1d6c801 --- /dev/null +++ b/tests/rdma_fake.cpp @@ -0,0 +1,403 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "rdma_fake.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct CmEvent { + rdma_cm_event event{}; + std::vector data; +}; +struct Id { + bool listening = false; + rdma_cm_id* peer = nullptr; +}; +struct Cq { + bool armed = false; + unsigned inject_qp = 0; + int polls = 0, notifications = 0; + std::deque completions; +}; +struct Fake { + std::unordered_map>> + channels; + std::unordered_map> delivered; + std::unordered_map ids; + std::unordered_map> contexts; + std::unordered_map> + completion_channels; + std::unordered_map cqs; + std::unordered_set pds; + std::unordered_set qps; + bool suppress_establish = false, suppress_connect = false, fail_qp = false, + fail_notify = false; + unsigned qp_extra = 0, next_qp = 1; + std::uint16_t next_port = 10000; + int cq_acks = 0, async_acks = 0; +} f; +ibv_device device{}; +ibv_context cm_context{}; + +int EventFd() { return eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC | EFD_SEMAPHORE); } +void Notify(int fd) { + std::uint64_t one = 1; + EXPECT_EQ(write(fd, &one, sizeof(one)), sizeof(one)); +} +void Consume(int fd) { + std::uint64_t one; + EXPECT_EQ(read(fd, &one, sizeof(one)), sizeof(one)); +} +int Empty() { + errno = EAGAIN; + return -1; +} +void Event(rdma_cm_id* id, rdma_cm_event_type type, + rdma_cm_id* listener = nullptr, const void* data = nullptr, + std::size_t length = 0, int status = 0) { + auto e = std::make_unique(); + e->event.id = id; + e->event.listen_id = listener; + e->event.event = type; + e->event.status = status; + if (data && length) { + auto* first = static_cast(data); + e->data.assign(first, first + length); + e->event.param.conn.private_data = e->data.data(); + e->event.param.conn.private_data_len = static_cast(length); + } + e->event.param.conn.initiator_depth = + e->event.param.conn.responder_resources = 1; + f.channels.at(id->channel).push_back(std::move(e)); + Notify(id->channel->fd); +} +int PollCq(ibv_cq* cq, int capacity, ibv_wc* output) { + auto& state = f.cqs.at(cq); + ++state.polls; + int count = 0; + while (count < capacity && !state.completions.empty()) { + output[count++] = state.completions.front(); + state.completions.pop_front(); + } + return count; +} +int ArmCq(ibv_cq* cq, int solicited) { + EXPECT_EQ(solicited, 0); + if (std::exchange(f.fail_notify, false)) return EIO; + auto& state = f.cqs.at(cq); + state.armed = true; + if (auto qp = std::exchange(state.inject_qp, 0U)) + rdma_fake::Completion(cq, qp); + return 0; +} +} // namespace + +namespace rdma_fake { +void Reset() { + f = {}; + cm_context.device = &device; +} +void CheckEmpty() { + EXPECT_TRUE(f.channels.empty()); + EXPECT_TRUE(f.delivered.empty()); + EXPECT_TRUE(f.ids.empty()); + EXPECT_TRUE(f.contexts.empty()); + EXPECT_TRUE(f.completion_channels.empty()); + EXPECT_TRUE(f.cqs.empty()); + EXPECT_TRUE(f.pds.empty()); + EXPECT_TRUE(f.qps.empty()); +} +void Cm(rdma_cm_id* id, rdma_cm_event_type event, int status) { + Event(id, event, nullptr, nullptr, 0, status); +} +void SuppressEstablish(bool value) { f.suppress_establish = value; } +void SuppressConnect(bool value) { f.suppress_connect = value; } +void RoundQp(unsigned extra) { f.qp_extra = extra; } +void FailNextQp() { f.fail_qp = true; } +void FailNextNotify() { f.fail_notify = true; } +void Completion(ibv_cq* cq, std::uint32_t qp, ibv_wc_status status) { + auto& state = f.cqs.at(cq); + ibv_wc wc{}; + wc.qp_num = qp; + wc.status = status; + state.completions.push_back(wc); + if (std::exchange(state.armed, false)) { + f.completion_channels.at(cq->channel).push_back(cq); + ++state.notifications; + Notify(cq->channel->fd); + } +} +void Async(ibv_context* context, ibv_event_type type, ibv_qp* qp) { + ibv_async_event e{}; + e.event_type = type; + if (type == IBV_EVENT_CQ_ERR) + e.element.cq = qp->send_cq; + else if (type != IBV_EVENT_DEVICE_FATAL) + e.element.qp = qp; + f.contexts.at(context).push_back(e); + Notify(context->async_fd); +} +void CompleteOnArm(ibv_cq* cq, std::uint32_t qp) { + f.cqs.at(cq).inject_qp = qp; +} +int CqPolls(ibv_cq* cq) { return f.cqs.at(cq).polls; } +int Notifications(ibv_cq* cq) { return f.cqs.at(cq).notifications; } +int AckedCqEvents() { return f.cq_acks; } +int AckedAsyncEvents() { return f.async_acks; } +int LiveQps() { return static_cast(f.qps.size()); } +int LiveIds() { return static_cast(f.ids.size()); } +} // namespace rdma_fake + +extern "C" { +rdma_event_channel* __wrap_rdma_create_event_channel() { + auto* c = new rdma_event_channel{}; + c->fd = EventFd(); + f.channels.emplace(c, std::deque>{}); + return c; +} +void __wrap_rdma_destroy_event_channel(rdma_event_channel* c) { + EXPECT_TRUE(f.channels.at(c).empty()); + f.channels.erase(c); + close(c->fd); + delete c; +} +int __wrap_rdma_create_id(rdma_event_channel* channel, rdma_cm_id** output, + void* context, rdma_port_space ps) { + EXPECT_EQ(ps, RDMA_PS_TCP); + auto* id = new rdma_cm_id{}; + id->channel = channel; + id->context = context; + id->verbs = &cm_context; + id->port_num = 1; + id->qp_type = IBV_QPT_RC; + f.ids.emplace(id, Id{}); + *output = id; + return 0; +} +int __wrap_rdma_destroy_id(rdma_cm_id* id) { + for (const auto& [key, event] : f.delivered) { + EXPECT_NE(event->event.id, id); + EXPECT_NE(event->event.listen_id, id); + } + auto& queue = f.channels.at(id->channel); + for (auto it = queue.begin(); it != queue.end();) { + if ((*it)->event.id == id) { + Consume(id->channel->fd); + it = queue.erase(it); + } else + ++it; + } + for (auto& [key, value] : f.ids) + if (value.peer == id) value.peer = nullptr; + EXPECT_EQ(f.ids.erase(id), 1U); + delete id; + return 0; +} +int __wrap_rdma_bind_addr(rdma_cm_id* id, sockaddr* address) { + auto* local = reinterpret_cast(&id->route.addr.src_addr); + *local = *reinterpret_cast(address); + if (!local->sin_port) local->sin_port = htons(f.next_port++); + return 0; +} +int __wrap_rdma_listen(rdma_cm_id* id, int) { + f.ids.at(id).listening = true; + return 0; +} +int __wrap_rdma_resolve_addr(rdma_cm_id* id, sockaddr*, sockaddr* destination, + int) { + *reinterpret_cast(&id->route.addr.dst_addr) = + *reinterpret_cast(destination); + Event(id, RDMA_CM_EVENT_ADDR_RESOLVED); + return 0; +} +int __wrap_rdma_resolve_route(rdma_cm_id* id, int) { + Event(id, RDMA_CM_EVENT_ROUTE_RESOLVED); + return 0; +} +int __wrap_rdma_connect(rdma_cm_id* id, rdma_conn_param* param) { + EXPECT_EQ(id->qp, nullptr); + EXPECT_NE(param->qp_num, 0U); + if (f.suppress_connect) return 0; + rdma_cm_id* listener = nullptr; + for (const auto& [key, value] : f.ids) + if (value.listening && rdma_get_src_port(key) == rdma_get_dst_port(id)) { + listener = key; + break; + } + if (!listener) { + Event(id, RDMA_CM_EVENT_REJECTED, nullptr, nullptr, 0, ECONNREFUSED); + return 0; + } + rdma_cm_id* accepted; + __wrap_rdma_create_id(listener->channel, &accepted, nullptr, RDMA_PS_TCP); + f.ids.at(accepted).peer = id; + f.ids.at(id).peer = accepted; + Event(accepted, RDMA_CM_EVENT_CONNECT_REQUEST, listener, param->private_data, + param->private_data_len); + return 0; +} +int __wrap_rdma_accept(rdma_cm_id* id, rdma_conn_param* param) { + EXPECT_EQ(id->qp, nullptr); + EXPECT_NE(param->qp_num, 0U); + Event(f.ids.at(id).peer, RDMA_CM_EVENT_CONNECT_RESPONSE, nullptr, + param->private_data, param->private_data_len); + return 0; +} +int __wrap_rdma_establish(rdma_cm_id* id) { + if (f.suppress_establish) return 0; + Event(f.ids.at(id).peer, RDMA_CM_EVENT_ESTABLISHED); + return 0; +} +int __wrap_rdma_disconnect(rdma_cm_id* id) { + if (auto* peer = f.ids.at(id).peer) Event(peer, RDMA_CM_EVENT_DISCONNECTED); + return 0; +} +int __wrap_rdma_reject(rdma_cm_id* id, const void*, std::uint8_t) { + if (auto* peer = f.ids.at(id).peer) + Event(peer, RDMA_CM_EVENT_REJECTED, nullptr, nullptr, 0, ECONNREFUSED); + return 0; +} +int __wrap_rdma_get_cm_event(rdma_event_channel* channel, + rdma_cm_event** output) { + auto& queue = f.channels.at(channel); + if (queue.empty()) return Empty(); + Consume(channel->fd); + auto event = std::move(queue.front()); + queue.pop_front(); + *output = &event->event; + f.delivered.emplace(*output, std::move(event)); + return 0; +} +int __wrap_rdma_ack_cm_event(rdma_cm_event* event) { + EXPECT_EQ(f.delivered.erase(event), 1U); + return 0; +} +int __wrap_rdma_init_qp_attr(rdma_cm_id*, ibv_qp_attr*, int* mask) { + *mask = IBV_QP_STATE; + return 0; +} +const char* __wrap_ibv_get_device_name(ibv_device*) { return "fake0"; } +ibv_context* __wrap_ibv_open_device(ibv_device* dev) { + auto* context = new ibv_context{}; + context->device = dev; + context->async_fd = EventFd(); + context->ops.poll_cq = PollCq; + context->ops.req_notify_cq = ArmCq; + f.contexts.emplace(context, std::deque{}); + return context; +} +int __wrap_ibv_close_device(ibv_context* context) { + EXPECT_TRUE(f.contexts.at(context).empty()); + f.contexts.erase(context); + close(context->async_fd); + delete context; + return 0; +} +int __wrap_ibv_query_device(ibv_context*, ibv_device_attr* attr) { + *attr = {}; + attr->max_cqe = 65536; + attr->max_qp_wr = 8192; + return 0; +} +ibv_pd* __wrap_ibv_alloc_pd(ibv_context* context) { + auto* pd = new ibv_pd{}; + pd->context = context; + f.pds.insert(pd); + return pd; +} +int __wrap_ibv_dealloc_pd(ibv_pd* pd) { + EXPECT_EQ(f.pds.erase(pd), 1U); + delete pd; + return 0; +} +ibv_comp_channel* __wrap_ibv_create_comp_channel(ibv_context* context) { + auto* channel = new ibv_comp_channel{}; + channel->context = context; + channel->fd = EventFd(); + f.completion_channels.emplace(channel, std::deque{}); + return channel; +} +int __wrap_ibv_destroy_comp_channel(ibv_comp_channel* channel) { + EXPECT_TRUE(f.completion_channels.at(channel).empty()); + f.completion_channels.erase(channel); + close(channel->fd); + delete channel; + return 0; +} +ibv_cq* __wrap_ibv_create_cq(ibv_context* context, int cqe, void* opaque, + ibv_comp_channel* channel, int) { + auto* cq = new ibv_cq{}; + cq->context = context; + cq->cq_context = opaque; + cq->cqe = cqe; + cq->channel = channel; + f.cqs.emplace(cq, Cq{}); + return cq; +} +int __wrap_ibv_destroy_cq(ibv_cq* cq) { + EXPECT_EQ(f.cqs.erase(cq), 1U); + delete cq; + return 0; +} +ibv_qp* __wrap_ibv_create_qp(ibv_pd* pd, ibv_qp_init_attr* attr) { + if (std::exchange(f.fail_qp, false)) { + errno = ENOMEM; + return nullptr; + } + EXPECT_EQ(attr->qp_type, IBV_QPT_RC); + EXPECT_EQ(pd->context, attr->send_cq->context); + EXPECT_EQ(pd->context, attr->recv_cq->context); + attr->cap.max_send_wr += f.qp_extra; + auto* qp = new ibv_qp{}; + qp->pd = pd; + qp->context = pd->context; + qp->qp_num = f.next_qp++; + qp->qp_type = attr->qp_type; + qp->send_cq = attr->send_cq; + qp->recv_cq = attr->recv_cq; + f.qps.insert(qp); + return qp; +} +int __wrap_ibv_modify_qp(ibv_qp* qp, ibv_qp_attr* attr, int) { + qp->state = attr->qp_state; + return 0; +} +int __wrap_ibv_destroy_qp(ibv_qp* qp) { + EXPECT_EQ(f.qps.erase(qp), 1U); + delete qp; + return 0; +} +int __wrap_ibv_get_cq_event(ibv_comp_channel* channel, ibv_cq** cq, + void** opaque) { + auto& queue = f.completion_channels.at(channel); + if (queue.empty()) return Empty(); + Consume(channel->fd); + *cq = queue.front(); + *opaque = (*cq)->cq_context; + queue.pop_front(); + return 0; +} +void __wrap_ibv_ack_cq_events(ibv_cq*, unsigned count) { f.cq_acks += count; } +int __wrap_ibv_get_async_event(ibv_context* context, ibv_async_event* event) { + auto& queue = f.contexts.at(context); + if (queue.empty()) return Empty(); + Consume(context->async_fd); + *event = queue.front(); + queue.pop_front(); + return 0; +} +void __wrap_ibv_ack_async_event(ibv_async_event*) { ++f.async_acks; } +} // extern "C" diff --git a/tests/rdma_fake.h b/tests/rdma_fake.h new file mode 100644 index 0000000..bb3e8a4 --- /dev/null +++ b/tests/rdma_fake.h @@ -0,0 +1,29 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include +#include + +#include + +namespace rdma_fake { +void Reset(); +void CheckEmpty(); +void SuppressConnect(bool suppress); +void SuppressEstablish(bool suppress); +void Cm(rdma_cm_id* id, rdma_cm_event_type event, int status = 0); +void RoundQp(unsigned extra); +void FailNextQp(); +void FailNextNotify(); +void Completion(ibv_cq* cq, std::uint32_t qp, + ibv_wc_status status = IBV_WC_SUCCESS); +void Async(ibv_context* context, ibv_event_type type, ibv_qp* qp = nullptr); +// Inject a CQE exactly when notification is requested, before the CQ recheck. +void CompleteOnArm(ibv_cq* cq, std::uint32_t qp); +int CqPolls(ibv_cq* cq); +int Notifications(ibv_cq* cq); +int AckedCqEvents(); +int AckedAsyncEvents(); +int LiveQps(); +int LiveIds(); +} // namespace rdma_fake diff --git a/tests/rdma_rc_test.cpp b/tests/rdma_rc_test.cpp new file mode 100644 index 0000000..0a7e5b2 --- /dev/null +++ b/tests/rdma_rc_test.cpp @@ -0,0 +1,472 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include + +#include "../src/rdma/rc_internal.h" +#include "bycorf/rdma/rc.h" +#include "bycorf/runtime/worker.h" +#ifdef BYCORF_RDMA_FAKE +#include "rdma_fake.h" +#define RcLoop RcLoopFake +#endif + +namespace bycorf::rdma::detail { +class RcTestPeer { + public: + static std::shared_ptr State(RcConnection& c) { return c.state_; } + static std::shared_ptr State(RcListener& l) { return l.state_; } +}; +} // namespace bycorf::rdma::detail +namespace { +using namespace bycorf; +using namespace bycorf::rdma; +using Peer = bycorf::rdma::detail::RcTestPeer; + +template +Task Collect(Task task, std::optional* result) { + result->emplace(co_await std::move(task)); + co_return absl::OkStatus(); +} +class RcLoop : public testing::Test { + protected: + void SetUp() override { +#ifdef BYCORF_RDMA_FAKE + rdma_fake::Reset(); + ip = "192.0.2.1"; +#else + ip = std::getenv("BYCORF_RDMA_TEST_IP"); + if (!ip || !*ip) GTEST_SKIP() << "Set BYCORF_RDMA_TEST_IP to opt in"; +#endif + cc.mailbox(0).wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(cc.mailbox(0).wake_fd_, 0); + worker.BindCrossCore(0, &cc); + SetThisWorker(0, &cc, &worker); + ASSERT_TRUE(worker.Init({.recv_buffer_count_ = 0}).ok()); + alarm(15); + initialized = true; + } + void TearDown() override { + if (!initialized) return; + alarm(0); + worker.Shutdown(); + worker.DestroyDetachedTasks(); + MutableThisWorker() = {}; + close(cc.mailbox(0).wake_fd_); +#ifdef BYCORF_RDMA_FAKE + rdma_fake::CheckEmpty(); +#endif + } + template + void Until(Predicate done) { + while (!done()) ASSERT_TRUE(worker.RunOnce(true)); + } + template + T Run(Task task) { + std::optional result; + worker.Spawn(Collect(std::move(task), &result)); + while (!result) worker.RunOnce(true); + return std::move(*result); + } + absl::StatusOr> Pair( + RcOptions options = {}) { + auto listener = RcListener::Listen(worker, ip, 0, options); + if (!listener.ok()) return listener.status(); + auto client = + Run(RcConnection::Connect(worker, ip, listener->port(), options)); + if (!client.ok()) return client.status(); + auto server = Run(listener->Accept()); + if (!server.ok()) return server.status(); + auto status = Run(listener->Close()); + if (!status.ok()) return status; + return std::pair(std::move(*client), std::move(*server)); + } + const char* ip = nullptr; + bool initialized = false; + CrossCore cc{1}; + Worker worker; +}; + +TEST_F(RcLoop, IdleConnectAcceptPrivateDataAndSharedCq) { + const std::array data{std::byte{0x31}, std::byte{0x72}}; + auto listening = RcListener::Listen(worker, ip, 0, {}, data); + ASSERT_TRUE(listening.ok()) << listening.status(); + auto listener = std::move(*listening); + std::optional> connected, accepted; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener.port(), {}, data), + &connected)); + worker.Spawn(Collect(listener.Accept(), &accepted)); + Until([&] { return connected && accepted; }); + ASSERT_TRUE(connected->ok()) << connected->status(); + ASSERT_TRUE(accepted->ok()) << accepted->status(); + auto client = std::move(**connected), server = std::move(**accepted); + EXPECT_TRUE(client.IsOpen()); + EXPECT_TRUE(server.IsOpen()); + ASSERT_GE(client.peer_private_data().size(), data.size()); + ASSERT_GE(server.peer_private_data().size(), data.size()); + EXPECT_TRUE( + std::equal(data.begin(), data.end(), client.peer_private_data().begin())); + EXPECT_TRUE( + std::equal(data.begin(), data.end(), server.peer_private_data().begin())); + EXPECT_EQ(client.protection_domain().native_pd(), + server.protection_domain().native_pd()); + auto a = Peer::State(client), b = Peer::State(server); + EXPECT_EQ(a->device->cq, b->device->cq); + EXPECT_EQ(a->qp->qp_type, IBV_QPT_RC); + EXPECT_EQ(a->qp->pd, client.protection_domain().native_pd()); + EXPECT_NE(a->id->verbs, a->qp->context); + auto device = a->device; + EXPECT_EQ(device->reserved, a->budget + b->budget); + EXPECT_TRUE(Run(listener.Close()).ok()); + EXPECT_TRUE(server.IsOpen()); + EXPECT_TRUE(Run(client.Close()).ok()); + EXPECT_TRUE(a->closed); + EXPECT_EQ(a->id, nullptr); + EXPECT_EQ(a->qp, nullptr); + Until([&] { return b->closed; }); + EXPECT_FALSE(server.IsOpen()); + EXPECT_FALSE(server.status().ok()); + EXPECT_EQ(device->reserved, 0); + EXPECT_TRUE(Run(server.Close()).ok()); +} + +TEST_F(RcLoop, CloseWakesAcceptAndShutdownRetiresPolls) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()) << listener.status(); + std::optional> accepted; + worker.Spawn(Collect(listener->Accept(), &accepted)); + worker.RunOnce(false); + auto state = Peer::State(*listener); + auto engine = state->engine.lock(); + ASSERT_TRUE(engine->cm_poll.armed); + EXPECT_TRUE(Run(listener->Close()).ok()); + Until([&] { return accepted.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(accepted->status())); + engine->BeginShutdown(); + EXPECT_TRUE(engine->cm_poll.armed) + << "cancel request must retain the poll tag"; + worker.Shutdown(); + EXPECT_FALSE(engine->cm_poll.armed); + EXPECT_EQ(engine->cm, nullptr); +} + +TEST_F(RcLoop, RefusedConnectionReportsCmError) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + const auto port = listener->port(); + ASSERT_TRUE(Run(listener->Close()).ok()); + auto result = Run(RcConnection::Connect(worker, ip, port)); + EXPECT_FALSE(result.ok()); + EXPECT_NE(std::string(result.status().message()).find("RDMA_CM_EVENT_"), + std::string::npos) + << result.status(); +} + +TEST_F(RcLoop, SeparateWorkersOwnDistinctContextsAndAcceptedConnections) { + CrossCore other_cc{1}; + Worker other; + other_cc.mailbox(0).wake_fd_ = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(other_cc.mailbox(0).wake_fd_, 0); + other.BindCrossCore(0, &other_cc); + std::optional> client, server; + auto listener = RcListener::Listen(worker, ip, 0); + // On assertion failure quiesce both workers before result/Task destruction. + struct Cleanup { + Worker& a; + Worker& b; + CrossCore& ca; + CrossCore& cb; + ~Cleanup() { + SetThisWorker(0, &cb, &b); + b.Shutdown(); + b.DestroyDetachedTasks(); + close(cb.mailbox(0).wake_fd_); + SetThisWorker(0, &ca, &a); + a.Shutdown(); + a.DestroyDetachedTasks(); + } + } cleanup{worker, other, cc, other_cc}; + ASSERT_TRUE(listener.ok()); + worker.Spawn(Collect(listener->Accept(), &server)); + SetThisWorker(0, &other_cc, &other); + ASSERT_TRUE(other.Init({.recv_buffer_count_ = 0}).ok()); + other.Spawn( + Collect(RcConnection::Connect(other, ip, listener->port()), &client)); + while (!client || !server) { + SetThisWorker(0, &cc, &worker); + worker.RunOnce(false); + SetThisWorker(0, &other_cc, &other); + other.RunOnce(false); + } + ASSERT_TRUE(client->ok()) << client->status(); + ASSERT_TRUE(server->ok()) << server->status(); + auto a = Peer::State(**client), b = Peer::State(**server); + EXPECT_EQ(a->engine.lock()->worker, &other); + EXPECT_EQ(b->engine.lock()->worker, &worker); + EXPECT_NE(a->qp->context, b->qp->context); + EXPECT_NE(a->device->cq, b->device->cq); + SetThisWorker(0, &cc, &worker); + worker.Shutdown(); + EXPECT_TRUE(b->closed); + SetThisWorker(0, &other_cc, &other); + while (!a->closed) other.RunOnce(true); + SetThisWorker(0, &cc, &worker); +} + +TEST_F(RcLoop, ConcurrentAcceptFailsAndCloseWakesOriginalWaiter) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + std::optional> first; + worker.Spawn(Collect(listener->Accept(), &first)); + worker.RunOnce(false); + auto second = Run(listener->Accept()); + EXPECT_TRUE(absl::IsFailedPrecondition(second.status())); + EXPECT_TRUE(Run(listener->Close()).ok()); + Until([&] { return first.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(first->status())); +} + +TEST_F(RcLoop, InvalidOptionsAndPrivateDataAreRejected) { + RcOptions options; + options.connect_timeout = std::chrono::milliseconds(0); + EXPECT_TRUE(absl::IsInvalidArgument( + Run(RcConnection::Connect(worker, ip, 123, options)).status())); + options = {}; + options.send_wr = 0; + EXPECT_TRUE(absl::IsInvalidArgument( + RcListener::Listen(worker, ip, 0, options).status())); + std::array oversized{}; + EXPECT_TRUE(absl::IsInvalidArgument( + RcListener::Listen(worker, ip, 0, {}, oversized).status())); + EXPECT_TRUE(absl::IsInvalidArgument( + Run(RcConnection::Connect(worker, "hostname", 123)).status())); +} + +#ifdef BYCORF_RDMA_FAKE +TEST_F(RcLoop, NotifyRecheckRaceAndRepeatedIdleNotifications) { + auto pair = Pair(); + ASSERT_TRUE(pair.ok()) << pair.status(); + auto state = Peer::State(pair->first); + auto* cq = state->device->cq; + const int polls = rdma_fake::CqPolls(cq); + rdma_fake::CompleteOnArm(cq, state->qp->qp_num); + rdma_fake::Completion(cq, state->qp->qp_num); + Until([&] { return rdma_fake::AckedCqEvents() >= 2; }); + EXPECT_GE(rdma_fake::CqPolls(cq), polls + 2); + EXPECT_EQ(rdma_fake::Notifications(cq), 2); + rdma_fake::Completion(cq, state->qp->qp_num); + Until([&] { return rdma_fake::AckedCqEvents() == 3; }); + EXPECT_TRUE(pair->first.IsOpen()); +} + +TEST_F(RcLoop, TimeoutRetiresIdQpAndTimerBeforeReturn) { + rdma_fake::SuppressConnect(true); + RcOptions options; + options.connect_timeout = std::chrono::milliseconds(10); + auto result = Run(RcConnection::Connect(worker, ip, 23456, options)); + EXPECT_TRUE(absl::IsDeadlineExceeded(result.status())) << result.status(); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + rdma_fake::SuppressConnect(false); + EXPECT_TRUE(Pair().ok()); +} + +TEST_F(RcLoop, SharedCqCapacityReleasedOnlyAfterQpRetirement) { + RcOptions options; + options.send_wr = options.recv_wr = 4; + options.cq_entries = 16; + auto first = Pair(options); + ASSERT_TRUE(first.ok()) << first.status(); + auto device = Peer::State(first->first)->device; + EXPECT_EQ(device->reserved, 16); + auto exhausted = Run(RcConnection::Connect(worker, ip, 12345, options)); + EXPECT_TRUE(absl::IsResourceExhausted(exhausted.status())) + << exhausted.status(); + EXPECT_EQ(device->reserved, 16); + EXPECT_TRUE(Run(first->first.Close()).ok()); + EXPECT_TRUE(Run(first->second.Close()).ok()); + EXPECT_EQ(device->reserved, 0); + EXPECT_TRUE(Pair(options).ok()); +} + +TEST_F(RcLoop, ProviderRoundedQpCapacityCannotOvercommitCq) { + rdma_fake::RoundQp(9); + RcOptions options; + options.send_wr = options.recv_wr = 4; + options.cq_entries = 16; + auto result = Run(RcConnection::Connect(worker, ip, 12345, options)); + EXPECT_TRUE(absl::IsResourceExhausted(result.status())) << result.status(); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_EQ(rdma_fake::LiveIds(), 0); +} + +TEST_F(RcLoop, FailedQpCreationUnwindsAndCanRetry) { + rdma_fake::FailNextQp(); + auto result = Run(RcConnection::Connect(worker, ip, 12345)); + EXPECT_FALSE(result.ok()); + EXPECT_NE(std::string(result.status().message()).find("ibv_create_qp"), + std::string::npos); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_TRUE(Pair().ok()); +} + +TEST_F(RcLoop, FailedNotificationSetupRetainsNoConnection) { + rdma_fake::FailNextNotify(); + auto result = Run(RcConnection::Connect(worker, ip, 12345)); + EXPECT_FALSE(result.ok()); + EXPECT_EQ(rdma_fake::LiveIds(), 0); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, CqFailureTerminatesAllSharingConnectionsAfterAck) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_CQ_ERR, state->qp); + Until([&] { return !first->first.IsOpen() && !second->first.IsOpen(); }); + EXPECT_EQ(rdma_fake::AckedAsyncEvents(), 1); + EXPECT_FALSE(first->second.IsOpen()); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_EQ(rdma_fake::LiveQps(), 0); + EXPECT_NE(std::string(second->first.status().message()) + .find(ibv_event_type_str(IBV_EVENT_CQ_ERR)), + std::string::npos); +} + +TEST_F(RcLoop, DeviceRemovalWithStatusFailsEveryAffectedConnection) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Cm(state->id, RDMA_CM_EVENT_DEVICE_REMOVAL, EIO); + Until([&] { return !first->first.IsOpen() && !second->first.IsOpen(); }); + EXPECT_FALSE(first->second.IsOpen()); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, QpFailureLeavesOtherConnectionsUsable) { + auto first = Pair(), second = Pair(); + ASSERT_TRUE(first.ok()); + ASSERT_TRUE(second.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_QP_FATAL, state->qp); + Until([&] { return Peer::State(first->second)->closed; }); + EXPECT_EQ(rdma_fake::AckedAsyncEvents(), 1); + EXPECT_TRUE(second->first.IsOpen()); + EXPECT_TRUE(second->second.IsOpen()); +} + +TEST_F(RcLoop, WorkerShutdownRetainsSharedPollsUntilRingQuiescence) { + auto pair = Pair(); + ASSERT_TRUE(pair.ok()); + auto state = Peer::State(pair->first); + auto engine = state->engine.lock(); + auto device = state->device; + rdma_fake::Completion(device->cq, state->qp->qp_num); + engine->BeginShutdown(); + EXPECT_TRUE(engine->cm_poll.armed); + EXPECT_TRUE(device->completion_poll.armed); + EXPECT_TRUE(device->async_poll.armed); + EXPECT_NE(device->cq, nullptr); + worker.Shutdown(); + EXPECT_FALSE(engine->cm_poll.armed); + EXPECT_FALSE(device->completion_poll.armed); + EXPECT_FALSE(device->async_poll.armed); + EXPECT_TRUE(state->closed); + EXPECT_EQ(rdma_fake::AckedCqEvents(), 1); +} + +TEST_F(RcLoop, ListenerCloseJoinsHandshakingChildren) { + rdma_fake::SuppressEstablish(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto client = Run(RcConnection::Connect(worker, ip, listener->port())); + ASSERT_TRUE(client.ok()) << client.status(); + auto state = Peer::State(*listener); + ASSERT_EQ(state->children.size(), 1U); + auto pending = state->children.begin()->second; + ASSERT_TRUE(pending->timer.armed); + EXPECT_TRUE(Run(listener->Close()).ok()); + EXPECT_FALSE(pending->timer.armed); + EXPECT_TRUE(pending->closed); + EXPECT_EQ(pending->id, nullptr); + EXPECT_TRUE(state->children.empty()); + Until([&] { return Peer::State(*client)->closed; }); +} + +TEST_F(RcLoop, CancellationKeepsConnectionUntilTimerTerminalCompletion) { + rdma_fake::SuppressConnect(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto engine = Peer::State(*listener)->engine.lock(); + std::optional> result; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener->port()), &result)); + Until([&] { return rdma_fake::LiveQps() > 0; }); + auto state = engine->connections.begin()->second; + engine->Close(state, absl::CancelledError("test cancel")); + EXPECT_TRUE(state->timer.armed); + EXPECT_NE(state->id, nullptr); + EXPECT_FALSE(state->closed); + // An event queued before the original poll retires must not resurrect it. + rdma_fake::Cm(state->id, RDMA_CM_EVENT_CONNECT_RESPONSE); + Until([&] { return result.has_value(); }); + EXPECT_TRUE(absl::IsCancelled(result->status())); + EXPECT_TRUE(state->closed); + EXPECT_EQ(rdma_fake::LiveQps(), 0); +} + +TEST_F(RcLoop, ShutdownWithSuspendedConnectAndAcceptReclaimsFrames) { + rdma_fake::SuppressConnect(true); + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto engine = Peer::State(*listener)->engine.lock(); + std::optional> connected, accepted; + worker.Spawn( + Collect(RcConnection::Connect(worker, ip, listener->port()), &connected)); + worker.Spawn(Collect(listener->Accept(), &accepted)); + Until([&] { return rdma_fake::LiveQps() > 0; }); + auto state = engine->connections.begin()->second; + worker.Shutdown(); + EXPECT_TRUE(state->closed); + EXPECT_FALSE(state->timer.armed); + EXPECT_TRUE(engine->connections.empty()); + worker.DestroyDetachedTasks(); + EXPECT_FALSE(state->connected.waiter); + EXPECT_FALSE(Peer::State(*listener)->accepted.waiter); + EXPECT_EQ(rdma_fake::LiveIds(), 0); +} + +TEST_F(RcLoop, DeviceFatalAndNotifyRearmFailureTerminateConnections) { + auto first = Pair(); + ASSERT_TRUE(first.ok()); + auto state = Peer::State(first->first); + rdma_fake::Async(state->qp->context, IBV_EVENT_DEVICE_FATAL); + Until([&] { return state->closed; }); + EXPECT_FALSE(first->second.IsOpen()); + // Drive terminal cancelled polls so failed device resources can retire. + auto engine = state->engine.lock(); + Until([&] { return engine->devices.empty(); }); + auto second = Pair(); + ASSERT_TRUE(second.ok()) << second.status(); + auto state2 = Peer::State(second->first); + rdma_fake::FailNextNotify(); + rdma_fake::Completion(state2->device->cq, state2->qp->qp_num); + Until([&] { return state2->closed; }); + EXPECT_FALSE(second->second.IsOpen()); + EXPECT_NE( + std::string(second->first.status().message()).find("ibv_req_notify_cq"), + std::string::npos); +} +#endif +} // namespace From 13307e6c48e5168a7735d7583c319175f4bb6cfd Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 15 Nov 2026 15:39:31 +0000 Subject: [PATCH 2/5] refactor(rdma): use flat child map and borrowed accept queue --- src/rdma/rc.cpp | 9 ++++-- src/rdma/rc_engine.cpp | 2 +- src/rdma/rc_internal.h | 10 +++++-- tests/rdma_rc_test.cpp | 65 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/rdma/rc.cpp b/src/rdma/rc.cpp index 9316bc2..93db9b3 100644 --- a/src/rdma/rc.cpp +++ b/src/rdma/rc.cpp @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -185,9 +186,13 @@ Task> RcListener::AcceptImpl( if (!status.ok()) co_return status; } if (state->closing) co_return state->error; - auto c = std::move(state->ready.front()); + auto child = state->children.find(state->ready.front()); + assert(child != state->children.end()); + // A failed connection may already be absent from the engine registry. Take + // the listener's ownership before removing either index, without suspending. + auto c = std::move(child->second); state->ready.pop_front(); - state->children.erase(c.get()); + state->children.erase(child); c->listener.reset(); if (!c->error.ok()) co_return c->error; co_return RcConnection(std::move(c)); diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp index 3766d50..5ac0eac 100644 --- a/src/rdma/rc_engine.cpp +++ b/src/rdma/rc_engine.cpp @@ -685,7 +685,7 @@ void Engine::Progress(std::shared_ptr c) { if (auto listener = c->listener.lock()) { if (!c->queued && !listener->closing) { c->queued = true; - listener->ready.push_back(c); + listener->ready.push_back(c.get()); listener->accepted.Publish(wake_worker()); } Progress(listener); diff --git a/src/rdma/rc_internal.h b/src/rdma/rc_internal.h index 16078d0..d22f47d 100644 --- a/src/rdma/rc_internal.h +++ b/src/rdma/rc_internal.h @@ -7,11 +7,12 @@ #include #include -#include #include #include #include +#include "absl/container/flat_hash_map.h" +#include "bycorf/base/inline_ring_queue.h" #include "bycorf/io/completion.h" #include "bycorf/rdma/rc.h" @@ -89,8 +90,11 @@ struct Listener { bool closing = false; bool closed = false; absl::Status error; - std::unordered_map> children; - std::deque> ready; + // Own unaccepted children even after failure removes them from the engine. + // ready only borrows these states: Accept transfers ownership before erasing + // a child, and listener retirement clears ready before releasing children. + absl::flat_hash_map> children; + InlineRingQueue ready; Signal accepted, retired; }; diff --git a/tests/rdma_rc_test.cpp b/tests/rdma_rc_test.cpp index 0a7e5b2..b117293 100644 --- a/tests/rdma_rc_test.cpp +++ b/tests/rdma_rc_test.cpp @@ -250,6 +250,71 @@ TEST_F(RcLoop, InvalidOptionsAndPrivateDataAreRejected) { } #ifdef BYCORF_RDMA_FAKE +TEST_F(RcLoop, QueuedAcceptPreservesOrderAndRetainsFailedResults) { + RcOptions options; + options.send_wr = options.recv_wr = 4; + auto listener = RcListener::Listen(worker, ip, 0, options); + ASSERT_TRUE(listener.ok()); + auto state = Peer::State(*listener); + auto engine = state->engine.lock(); + std::vector clients; + std::vector> queued; + // Queue a burst before Accept so both the child table and FIFO must grow. + for (unsigned i = 0; i < 12; ++i) { + const std::array data{static_cast(i)}; + auto client = + Run(RcConnection::Connect(worker, ip, listener->port(), options, data)); + ASSERT_TRUE(client.ok()) << client.status(); + clients.push_back(std::move(*client)); + Until([&] { return state->ready.size() == i + 1; }); + queued.push_back(state->children.at(state->ready[i])); + } + { + auto failed = queued[3].lock(); + ASSERT_TRUE(failed); + const auto id = failed->id; + rdma_fake::Cm(id, RDMA_CM_EVENT_DISCONNECTED); + Until([&] { return failed->closed; }); + EXPECT_FALSE(engine->connections.contains(id)); + } + // Only the listener retains this retired state until Accept reads its error. + EXPECT_FALSE(queued[3].expired()); + std::vector accepted; + for (unsigned i = 0; i < queued.size(); ++i) { + auto connection = Run(listener->Accept()); + if (i == 3) { + EXPECT_TRUE(absl::IsUnavailable(connection.status())); + EXPECT_TRUE(queued[i].expired()); + } else { + ASSERT_TRUE(connection.ok()) << connection.status(); + ASSERT_FALSE(connection->peer_private_data().empty()); + EXPECT_EQ(connection->peer_private_data().front(), + static_cast(i)); + accepted.push_back(std::move(*connection)); + } + } + EXPECT_TRUE(state->ready.empty()); + EXPECT_TRUE(state->children.empty()); + EXPECT_TRUE(Run(listener->Close()).ok()); + for (const auto& connection : accepted) EXPECT_TRUE(connection.IsOpen()); +} + +TEST_F(RcLoop, ListenerCloseReleasesUnacceptedReadyStates) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + auto state = Peer::State(*listener); + auto client = Run(RcConnection::Connect(worker, ip, listener->port())); + ASSERT_TRUE(client.ok()) << client.status(); + Until([&] { return !state->ready.empty(); }); + std::weak_ptr queued = + state->children.at(state->ready.front()); + EXPECT_TRUE(Run(listener->Close()).ok()); + EXPECT_TRUE(state->ready.empty()); + EXPECT_TRUE(state->children.empty()); + EXPECT_TRUE(queued.expired()); + EXPECT_TRUE(absl::IsCancelled(Run(listener->Accept()).status())); +} + TEST_F(RcLoop, NotifyRecheckRaceAndRepeatedIdleNotifications) { auto pair = Pair(); ASSERT_TRUE(pair.ok()) << pair.status(); From 5274f42b50d49ea89ece7d774a579500c5217de7 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 17 Nov 2026 12:41:08 +0000 Subject: [PATCH 3/5] refactor(rdma): use thread-confined non-atomic ownership --- CMakeLists.txt | 1 + docs/architecture/rdma.md | 15 +- include/bycorf/base/local_shared_ptr.h | 242 +++++++++++++++++++++++++ include/bycorf/io/rdma_backend.h | 4 +- include/bycorf/io/rdma_reactor.h | 8 +- include/bycorf/rdma/rc.h | 18 +- src/rdma/rc.cpp | 12 +- src/rdma/rc_engine.cpp | 46 ++--- src/rdma/rc_internal.h | 42 +++-- tests/local_shared_ptr_test.cpp | 148 +++++++++++++++ tests/rdma_rc_test.cpp | 61 ++++++- 11 files changed, 528 insertions(+), 69 deletions(-) create mode 100644 include/bycorf/base/local_shared_ptr.h create mode 100644 tests/local_shared_ptr_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a7f3fc3..d587b8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -474,6 +474,7 @@ if (BYCORF_BUILD_TESTS) tests/cross_core_drain_test.cpp tests/worker_scheduler_test.cpp tests/received_buffer_queue_test.cpp + tests/local_shared_ptr_test.cpp tests/backend_selection_test.cpp) target_link_libraries(bycorf_unit_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_unit_tests 30 "runtime;unit") diff --git a/docs/architecture/rdma.md b/docs/architecture/rdma.md index f2f24ae..6419c4f 100644 --- a/docs/architecture/rdma.md +++ b/docs/architecture/rdma.md @@ -95,8 +95,13 @@ failed CM channel stops all endpoints on its worker and rejects new admission. ## Close and worker shutdown -Public handles are move-only and used on their owner worker. Destruction -requests close; awaiting Close joins retirement. Closing stops admission, +Public handles are move-only and used on their owner worker. Engine, device, +listener and connection states use non-atomic strong/weak ownership. Every +handle and associated coroutine frame is accessed, moved and destroyed on its +creating thread, including any handles retained after worker shutdown; they +cannot be handed to another thread for cleanup. Exported PD/MR/context handles +retain their separate shared ownership contract. Destruction requests close; +awaiting Close joins retirement. Closing stops admission, disconnects the CM association and puts the external QP into ERR. The connection retains its timer tag until the timer's terminal completion, even when cancellation has already been accepted. It leaves the ID registry before @@ -117,7 +122,11 @@ Only after ring shutdown confirms that no kernel operation can refer to those tags does RDMA retire requests that did not receive callbacks, acknowledge remaining events, destroy QPs/CM IDs, and release channels/CQs. Coroutine frames are reclaimed afterward. Exported registration/PD handles retain their own -context independently. Unexpected destruction failures fail closed rather than +context independently. `Server` quiesces the ring and reclaims coroutine frames +on the worker thread. Direct `Runtime` integrations must likewise call worker +`Shutdown()` and `DestroyDetachedTasks()` on that thread before it exits, and +release any application-owned RC handles and Tasks there. Unexpected destruction +failures fail closed rather than release resources that may still be referenced by the provider. ## Registered memory diff --git a/include/bycorf/base/local_shared_ptr.h b/include/bycorf/base/local_shared_ptr.h new file mode 100644 index 0000000..2748fd4 --- /dev/null +++ b/include/bycorf/base/local_shared_ptr.h @@ -0,0 +1,242 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#ifndef NDEBUG +#include +#endif + +namespace bycorf { +namespace local_shared_detail { +// Same allocating-thread contract as Lavik's LocalSharedPtr. The erased +// control also supports incomplete types and weak references for RC handles. +struct Control { + std::size_t strong = 1; + // One implicit weak reference belongs to the entire strong-owner cohort. + // It keeps the control alive while T's destructor releases its own weak refs. + std::size_t weak = 1; + void (*destroy)(Control*) noexcept; + void (*deallocate)(Control*) noexcept; +#ifndef NDEBUG + const std::thread::id owner = std::this_thread::get_id(); +#endif + void AssertOwner() const noexcept { +#ifndef NDEBUG + assert(owner == std::this_thread::get_id() && + "LocalSharedPtr used outside its owner thread"); +#endif + } + void RetainStrong() noexcept { + AssertOwner(); + assert(strong && strong != std::numeric_limits::max()); + ++strong; + } + void RetainWeak() noexcept { + AssertOwner(); + assert(weak && weak != std::numeric_limits::max()); + ++weak; + } + void ReleaseWeak() noexcept { + AssertOwner(); + if (--weak == 0) deallocate(this); + } + void ReleaseStrong() noexcept { + AssertOwner(); + if (--strong == 0) { + destroy(this); + ReleaseWeak(); + } + } +}; + +template +struct Allocation final : Control { + // Object destruction and allocation release happen at different times when + // weak observers survive. Keeping T in a union avoids double destruction. + union { + T value; + }; + template + explicit Allocation(Args&&... args) + : Control{.destroy = + [](Control* c) noexcept { + std::destroy_at(&static_cast(c)->value); + }, + .deallocate = + [](Control* c) noexcept { + delete static_cast(c); + }}, + value(std::forward(args)...) {} + ~Allocation() {} +}; +} // namespace local_shared_detail + +template +class LocalWeakPtr; +template +class LocalSharedPtr; +template +LocalSharedPtr MakeLocalShared(Args&&... args); + +// Non-atomic shared ownership confined to the allocating thread, including +// access, copying, moving and destruction, even after the worker shuts down. +// Coroutine frames carrying these handles must also be destroyed there. +// Debug builds check thread affinity before accessing the control block. +// Supports incomplete types and T -> const T; no aliasing, raw adoption, +// arrays, custom deleters or cross-thread handoff. Use MakeLocalShared. +template +class LocalSharedPtr { + static_assert(std::is_object_v && !std::is_array_v && + !std::is_volatile_v); + template + static constexpr bool kCompatible = + std::is_same_v, std::remove_const_t> && + std::is_convertible_v; + + public: + constexpr LocalSharedPtr() noexcept = default; + constexpr LocalSharedPtr(std::nullptr_t) noexcept {} + LocalSharedPtr(const LocalSharedPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainStrong(); + } + template + requires kCompatible + LocalSharedPtr(const LocalSharedPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainStrong(); + } + LocalSharedPtr(LocalSharedPtr&& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + other.control_ = nullptr; + other.value_ = nullptr; + } + ~LocalSharedPtr() { reset(); } + LocalSharedPtr& operator=(LocalSharedPtr other) noexcept { + swap(other); + return *this; + } + void reset() noexcept { + auto* control = CheckedControl(); + control_ = nullptr; + value_ = nullptr; + if (control) control->ReleaseStrong(); + } + void swap(LocalSharedPtr& other) noexcept { + (void)CheckedControl(); + (void)other.CheckedControl(); + std::swap(control_, other.control_); + std::swap(value_, other.value_); + } + T* get() const noexcept { + (void)CheckedControl(); + return value_; + } + T* operator->() const noexcept { + assert(value_); + return get(); + } + T& operator*() const noexcept { return *operator->(); } + explicit operator bool() const noexcept { return get() != nullptr; } + std::size_t use_count() const noexcept { + auto* control = CheckedControl(); + return control ? control->strong : 0; + } + friend bool operator==(const LocalSharedPtr& a, + const LocalSharedPtr& b) noexcept { + return a.get() == b.get(); + } + friend bool operator==(const LocalSharedPtr& a, std::nullptr_t) noexcept { + return !a; + } + + private: + template + friend class LocalSharedPtr; + template + friend class LocalWeakPtr; + template + friend LocalSharedPtr MakeLocalShared(Args&&...); + LocalSharedPtr(local_shared_detail::Control* control, T* value) noexcept + : control_(control), value_(value) {} + local_shared_detail::Control* CheckedControl() const noexcept { + if (control_) control_->AssertOwner(); + return control_; + } + local_shared_detail::Control* control_ = nullptr; + T* value_ = nullptr; +}; + +// Non-owning observation with the same thread confinement as LocalSharedPtr. +// lock() retains a live object; after the last strong release it returns empty. +// Weak handles keep only the allocation/control alive, never the object. +template +class LocalWeakPtr { + public: + constexpr LocalWeakPtr() noexcept = default; + LocalWeakPtr(const LocalSharedPtr& strong) noexcept + : control_(strong.CheckedControl()), value_(strong.value_) { + if (control_) control_->RetainWeak(); + } + LocalWeakPtr(const LocalWeakPtr& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + if (control_) control_->RetainWeak(); + } + LocalWeakPtr(LocalWeakPtr&& other) noexcept + : control_(other.CheckedControl()), value_(other.value_) { + other.control_ = nullptr; + other.value_ = nullptr; + } + ~LocalWeakPtr() { reset(); } + LocalWeakPtr& operator=(LocalWeakPtr other) noexcept { + swap(other); + return *this; + } + void swap(LocalWeakPtr& other) noexcept { + (void)CheckedControl(); + (void)other.CheckedControl(); + std::swap(control_, other.control_); + std::swap(value_, other.value_); + } + void reset() noexcept { + auto* control = CheckedControl(); + control_ = nullptr; + value_ = nullptr; + if (control) control->ReleaseWeak(); + } + bool expired() const noexcept { + auto* control = CheckedControl(); + return !control || control->strong == 0; + } + LocalSharedPtr lock() const noexcept { + if (expired()) return {}; + control_->RetainStrong(); + return LocalSharedPtr(control_, value_); + } + + private: + local_shared_detail::Control* CheckedControl() const noexcept { + if (control_) control_->AssertOwner(); + return control_; + } + local_shared_detail::Control* control_ = nullptr; + T* value_ = nullptr; +}; + +// Constructs object and control in one allocation. Constructor failure frees +// that allocation; T's destructor must not throw. The last strong release +// destroys T and the last weak release frees the allocation. +template +LocalSharedPtr MakeLocalShared(Args&&... args) { + auto* block = new local_shared_detail::Allocation>( + std::forward(args)...); + return LocalSharedPtr(block, std::addressof(block->value)); +} +} // namespace bycorf diff --git a/include/bycorf/io/rdma_backend.h b/include/bycorf/io/rdma_backend.h index 7bc90bc..54d88ab 100644 --- a/include/bycorf/io/rdma_backend.h +++ b/include/bycorf/io/rdma_backend.h @@ -70,7 +70,7 @@ class RdmaBackend { engine_.reset(); resources_.Shutdown(); } - absl::StatusOr> GetRdmaEngine() { + absl::StatusOr> GetRdmaEngine() { if (!engine_) { auto created = rdma::detail::CreateEngine(static_cast(*this)); if (!created.ok()) return created.status(); @@ -89,7 +89,7 @@ class RdmaBackend { return static_cast(*this).KernelBackend(); } detail::RdmaResources resources_; - std::shared_ptr engine_; + LocalSharedPtr engine_; }; } // namespace bycorf #endif // BYCORF_ENABLE_RDMA diff --git a/include/bycorf/io/rdma_reactor.h b/include/bycorf/io/rdma_reactor.h index 4d807ee..867810d 100644 --- a/include/bycorf/io/rdma_reactor.h +++ b/include/bycorf/io/rdma_reactor.h @@ -4,9 +4,9 @@ #pragma once #if BYCORF_ENABLE_RDMA -#include #include "absl/status/statusor.h" +#include "bycorf/base/local_shared_ptr.h" namespace bycorf { class Worker; @@ -17,9 +17,9 @@ namespace detail { class Engine; // Used only by the private worker facade. Shutdown has two phases because // io_uring drops outstanding tags without callbacks when the ring is destroyed. -absl::StatusOr> CreateEngine(Worker& worker); -void BeginShutdown(const std::shared_ptr& engine); -void AfterRingShutdown(const std::shared_ptr& engine); +absl::StatusOr> CreateEngine(Worker& worker); +void BeginShutdown(const LocalSharedPtr& engine); +void AfterRingShutdown(const LocalSharedPtr& engine); } // namespace detail } // namespace rdma } // namespace bycorf diff --git a/include/bycorf/rdma/rc.h b/include/bycorf/rdma/rc.h index e774261..eac7d59 100644 --- a/include/bycorf/rdma/rc.h +++ b/include/bycorf/rdma/rc.h @@ -8,11 +8,11 @@ #include #include -#include #include #include #include +#include "bycorf/base/local_shared_ptr.h" #include "bycorf/io/rdma_memory.h" #include "bycorf/runtime/task.h" @@ -39,7 +39,8 @@ struct RcOptions { // Owner-worker-only RC connection. Move-only; destruction requests asynchronous // close. Await Close() to join resource retirement while the worker is running. // Worker shutdown also closes all connections before reclaiming coroutine -// frames. +// frames. Handles and returned Tasks use non-atomic ownership: access, moves +// and destruction stay on the creating thread, including after shutdown. class RcConnection { public: RcConnection() = default; @@ -69,16 +70,17 @@ class RcConnection { private: friend class RcListener; friend class detail::RcTestPeer; - explicit RcConnection(std::shared_ptr state); + explicit RcConnection(LocalSharedPtr state); static Task> ConnectImpl( - std::shared_ptr engine, sockaddr_storage address, + LocalSharedPtr engine, sockaddr_storage address, RcOptions options, std::vector data); - std::shared_ptr state_; + LocalSharedPtr state_; }; // A listener and every accepted connection stay on the listening worker. // Accepted connections already returned to the application outlive the -// listener. +// listener. Handles and returned Tasks follow the same creating-thread lifetime +// contract as RcConnection, including destruction after worker shutdown. class RcListener { public: RcListener() = default; @@ -103,8 +105,8 @@ class RcListener { private: friend class detail::RcTestPeer; static Task> AcceptImpl( - std::shared_ptr state); - std::shared_ptr state_; + LocalSharedPtr state); + LocalSharedPtr state_; }; } // namespace rdma } // namespace bycorf diff --git a/src/rdma/rc.cpp b/src/rdma/rc.cpp index 93db9b3..cf7c130 100644 --- a/src/rdma/rc.cpp +++ b/src/rdma/rc.cpp @@ -51,7 +51,7 @@ Task> FailedConnect(absl::Status error) { } template -void Drop(const std::shared_ptr& state) { +void Drop(const LocalSharedPtr& state) { if (!state || state->closed) return; if (auto engine = state->engine.lock()) { // Destruction can request asynchronous close but cannot migrate ownership. @@ -61,7 +61,7 @@ void Drop(const std::shared_ptr& state) { } } template -Task CloseImpl(std::shared_ptr state) { +Task CloseImpl(LocalSharedPtr state) { if (!state || state->closed) co_return absl::OkStatus(); auto engine = state->engine.lock(); if (!engine) co_return absl::CancelledError("RDMA owner is gone"); @@ -71,7 +71,7 @@ Task CloseImpl(std::shared_ptr state) { co_return co_await detail::Wait{state->retired}; } struct ConnectGuard { - std::shared_ptr state; + LocalSharedPtr state; bool delivered = false; ~ConnectGuard() { if (!delivered) Drop(state); @@ -79,7 +79,7 @@ struct ConnectGuard { }; } // namespace -RcConnection::RcConnection(std::shared_ptr state) +RcConnection::RcConnection(LocalSharedPtr state) : state_(std::move(state)) {} RcConnection::RcConnection(RcConnection&&) noexcept = default; RcConnection& RcConnection::operator=(RcConnection&& other) noexcept { @@ -106,7 +106,7 @@ Task> RcConnection::Connect( {private_data.begin(), private_data.end()}); } Task> RcConnection::ConnectImpl( - std::shared_ptr engine, sockaddr_storage address, + LocalSharedPtr engine, sockaddr_storage address, RcOptions options, std::vector data) { if (!engine->OnOwner()) co_return absl::FailedPreconditionError( @@ -169,7 +169,7 @@ Task> RcListener::Accept() { return AcceptImpl(state_); } Task> RcListener::AcceptImpl( - std::shared_ptr state) { + LocalSharedPtr state) { if (!state) co_return absl::FailedPreconditionError("empty RDMA listener"); auto engine = state->engine.lock(); if (!engine || state->closing) diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp index 5ac0eac..df202a7 100644 --- a/src/rdma/rc_engine.cpp +++ b/src/rdma/rc_engine.cpp @@ -107,7 +107,7 @@ void Poll::Stop() { void Poll::Complete(Worker&, int result, unsigned) { // Failure fanout may remove the device cache entry. This event reference // keeps the containing poll alive until the callback has fully returned. - auto retained = device ? device->shared_from_this() : nullptr; + auto retained = device ? device->self.lock() : nullptr; armed = false; if (!stopping) engine->OnPoll(*this, result); // A cancellation CQE is the retirement boundary; cancellation admission @@ -149,30 +149,31 @@ absl::Status Engine::Init() { cm_poll.fd = cm->fd; return cm_poll.Arm(); } -absl::StatusOr> CreateEngine(Worker& worker) { +absl::StatusOr> CreateEngine(Worker& worker) { if (ThisWorker().self_ != &worker) return absl::FailedPreconditionError("RDMA must run on its owner worker"); - auto engine = std::make_shared(worker); + auto engine = MakeLocalShared(worker); + engine->self = engine; auto status = engine->Init(); if (!status.ok()) return status; return engine; } -void BeginShutdown(const std::shared_ptr& engine) { +void BeginShutdown(const LocalSharedPtr& engine) { if (engine) engine->BeginShutdown(); } -void AfterRingShutdown(const std::shared_ptr& engine) { +void AfterRingShutdown(const LocalSharedPtr& engine) { if (engine) engine->AfterRingShutdown(); } -absl::StatusOr> Engine::Listen( +absl::StatusOr> Engine::Listen( const sockaddr_storage& address, RcOptions options, std::vector data, int backlog) { if (stopping) return absl::CancelledError("RDMA worker is shutting down"); if (!OnOwner()) return absl::FailedPreconditionError("RDMA call on a different worker"); if (!error.ok()) return error; - auto listener = std::make_shared(); - listener->engine = shared_from_this(); + auto listener = MakeLocalShared(); + listener->engine = self; listener->options = options; listener->private_data = std::move(data); listener->backlog = backlog; @@ -190,7 +191,7 @@ absl::StatusOr> Engine::Listen( listeners.emplace(listener->id, listener); return listener; } -absl::Status Engine::StartTimer(const std::shared_ptr& c) { +absl::Status Engine::StartTimer(const LocalSharedPtr& c) { c->expires = std::chrono::steady_clock::now() + c->options.connect_timeout; c->timer.engine = this; c->timer.connection = c.get(); @@ -204,15 +205,15 @@ absl::Status Engine::StartTimer(const std::shared_ptr& c) { if (status.ok()) c->timer.armed = true; return status; } -absl::StatusOr> Engine::Connect( +absl::StatusOr> Engine::Connect( const sockaddr_storage& address, RcOptions options, std::vector data) { if (stopping) return absl::CancelledError("RDMA worker is shutting down"); if (!OnOwner()) return absl::FailedPreconditionError("RDMA call on a different worker"); if (!error.ok()) return error; - auto c = std::make_shared(); - c->engine = shared_from_this(); + auto c = MakeLocalShared(); + c->engine = self; c->options = options; c->local_data = std::move(data); if (rdma_create_id(cm, &c->id, nullptr, RDMA_PS_TCP)) @@ -228,7 +229,7 @@ absl::StatusOr> Engine::Connect( return c; } -absl::StatusOr> Engine::GetDevice(Connection& c) { +absl::StatusOr> Engine::GetDevice(Connection& c) { auto* selected = c.id->verbs; if (!selected || !selected->device) return absl::FailedPreconditionError("CM did not select a verbs device"); @@ -240,7 +241,8 @@ absl::StatusOr> Engine::GetDevice(Connection& c) { "shared RDMA CQ capacity is already fixed"); return it->second; } - auto device = std::make_shared(); + auto device = MakeLocalShared(); + device->self = device; auto* opened = ibv_open_device(selected->device); if (!opened) return ErrnoStatus("ibv_open_device(RC)"); device->context = @@ -299,7 +301,7 @@ absl::Status Engine::Transition(Connection& c, ibv_qp_state state) { return result ? absl::ErrnoToStatus(result, "ibv_modify_qp(RC)") : absl::OkStatus(); } -absl::Status Engine::CreateQp(const std::shared_ptr& c) { +absl::Status Engine::CreateQp(const LocalSharedPtr& c) { auto found = GetDevice(*c); if (!found.ok()) return found.status(); auto device = *found; @@ -335,7 +337,7 @@ absl::Status Engine::CreateQp(const std::shared_ptr& c) { return Transition(*c, IBV_QPS_INIT); } -void Engine::Established(const std::shared_ptr& c) { +void Engine::Established(const LocalSharedPtr& c) { if (std::chrono::steady_clock::now() >= c->expires) { Close(c, absl::DeadlineExceededError("RDMA connection deadline exceeded")); return; @@ -380,8 +382,8 @@ void Engine::HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, return; } auto listener = it->second; - auto c = std::make_shared(); - c->engine = shared_from_this(); + auto c = MakeLocalShared(); + c->engine = self; c->listener = listener; c->id = id; c->options = listener->options; @@ -641,7 +643,7 @@ void Engine::FailAll(const absl::Status& error) { for (auto& c : Snapshot(connections)) Close(c, error); } -void Engine::Close(std::shared_ptr c, absl::Status reason) { +void Engine::Close(LocalSharedPtr c, absl::Status reason) { if (c->closing || c->closed) return; c->closing = true; c->handshake_done = true; @@ -660,7 +662,7 @@ void Engine::Close(std::shared_ptr c, absl::Status reason) { } Progress(c); } -void Engine::Progress(std::shared_ptr c) { +void Engine::Progress(LocalSharedPtr c) { if (c->timer.armed || c->closed) return; if (c->closing) { // Erase before any CQ drain can fail another connection and reenter this @@ -692,7 +694,7 @@ void Engine::Progress(std::shared_ptr c) { } } } -void Engine::Close(std::shared_ptr listener, absl::Status reason) { +void Engine::Close(LocalSharedPtr listener, absl::Status reason) { if (listener->closing || listener->closed) return; listener->closing = true; listener->error = reason; @@ -704,7 +706,7 @@ void Engine::Close(std::shared_ptr listener, absl::Status reason) { listener->accepted.Publish(wake_worker()); Progress(listener); } -void Engine::Progress(std::shared_ptr listener) { +void Engine::Progress(LocalSharedPtr listener) { if (!listener->closing || listener->closed) return; for (const auto& [key, c] : listener->children) if (!c->closed) return; diff --git a/src/rdma/rc_internal.h b/src/rdma/rc_internal.h index d22f47d..7d19597 100644 --- a/src/rdma/rc_internal.h +++ b/src/rdma/rc_internal.h @@ -58,11 +58,11 @@ struct Deadline final : IoCompletion { }; struct Connection { - std::weak_ptr engine; - std::weak_ptr listener; + LocalWeakPtr engine; + LocalWeakPtr listener; rdma_cm_id* id = nullptr; ibv_qp* qp = nullptr; - std::shared_ptr device; + LocalSharedPtr device; std::uint64_t budget = 0; RcOptions options; std::vector local_data, peer_data; @@ -81,7 +81,7 @@ struct Connection { }; struct Listener { - std::weak_ptr engine; + LocalWeakPtr engine; rdma_cm_id* id = nullptr; RcOptions options; std::vector private_data; @@ -93,12 +93,13 @@ struct Listener { // Own unaccepted children even after failure removes them from the engine. // ready only borrows these states: Accept transfers ownership before erasing // a child, and listener retirement clears ready before releasing children. - absl::flat_hash_map> children; + absl::flat_hash_map> children; InlineRingQueue ready; Signal accepted, retired; }; -struct Device : std::enable_shared_from_this { +struct Device { + LocalWeakPtr self; std::shared_ptr context; RdmaProtectionDomain domain; ibv_comp_channel* channel = nullptr; @@ -112,21 +113,22 @@ struct Device : std::enable_shared_from_this { ~Device(); }; -class Engine : public std::enable_shared_from_this { +class Engine { public: + LocalWeakPtr self; explicit Engine(Worker& owner) : worker(&owner) {} ~Engine(); absl::Status Init(); - absl::StatusOr> Listen( + absl::StatusOr> Listen( const sockaddr_storage& address, RcOptions options, std::vector data, int backlog); - absl::StatusOr> Connect( + absl::StatusOr> Connect( const sockaddr_storage& address, RcOptions options, std::vector data); - void Close(std::shared_ptr connection, absl::Status reason); - void Close(std::shared_ptr listener, absl::Status reason); - void Progress(std::shared_ptr connection); - void Progress(std::shared_ptr listener); + void Close(LocalSharedPtr connection, absl::Status reason); + void Close(LocalSharedPtr listener, absl::Status reason); + void Progress(LocalSharedPtr connection); + void Progress(LocalSharedPtr listener); void OnPoll(Poll& poll, int result); void DrainCm(); void DrainCq(Device& device); @@ -143,18 +145,18 @@ class Engine : public std::enable_shared_from_this { absl::Status error; rdma_event_channel* cm = nullptr; Poll cm_poll; - std::unordered_map> connections; - std::unordered_map> listeners; - std::unordered_map> devices; + std::unordered_map> connections; + std::unordered_map> listeners; + std::unordered_map> devices; private: - absl::Status StartTimer(const std::shared_ptr& connection); - absl::Status CreateQp(const std::shared_ptr& connection); + absl::Status StartTimer(const LocalSharedPtr& connection); + absl::Status CreateQp(const LocalSharedPtr& connection); absl::Status Transition(Connection& connection, ibv_qp_state state); - absl::StatusOr> GetDevice(Connection& connection); + absl::StatusOr> GetDevice(Connection& connection); void HandleCm(rdma_cm_id* id, rdma_cm_id* listen_id, rdma_cm_event_type event, int status, std::vector data, rdma_conn_param peer); - void Established(const std::shared_ptr& connection); + void Established(const LocalSharedPtr& connection); void FailAll(const absl::Status& error); void DrainNotifications(Device& device, bool rearm); }; diff --git a/tests/local_shared_ptr_test.cpp b/tests/local_shared_ptr_test.cpp new file mode 100644 index 0000000..cd65eb9 --- /dev/null +++ b/tests/local_shared_ptr_test.cpp @@ -0,0 +1,148 @@ +// Copyright (C) 2026 EloqData Inc. +// SPDX-License-Identifier: Apache-2.0 + +#include "bycorf/base/local_shared_ptr.h" + +#include + +#include +#include + +namespace bycorf { +namespace { +struct Tracked { + explicit Tracked(int& destroyed) : destroyed(destroyed) {} + ~Tracked() { ++destroyed; } + int& destroyed; +}; + +TEST(LocalSharedPtr, StrongCopiesMovesAndConstConversion) { + int destroyed = 0; + auto first = MakeLocalShared(destroyed); + auto second = first; + LocalSharedPtr view = first; + EXPECT_EQ(first.use_count(), 3U); + auto moved = std::move(second); + EXPECT_FALSE(second); + EXPECT_EQ(moved.get(), first.get()); + EXPECT_EQ(first.use_count(), 3U); + first.reset(); + moved = nullptr; + EXPECT_EQ(destroyed, 0); + EXPECT_EQ(view.use_count(), 1U); + view.reset(); + EXPECT_EQ(destroyed, 1); +} + +TEST(LocalSharedPtr, WeakLockDoesNotResurrectDestroyedObject) { + int destroyed = 0; + auto first = MakeLocalShared(destroyed); + LocalWeakPtr weak = first; + auto copy = weak; + auto moved = std::move(copy); + EXPECT_TRUE(copy.expired()); + auto retained = moved.lock(); + first.reset(); + EXPECT_EQ(destroyed, 0); + EXPECT_FALSE(weak.expired()); + retained.reset(); + EXPECT_EQ(destroyed, 1); + EXPECT_TRUE(weak.expired()); + EXPECT_FALSE(moved.lock()); + weak.reset(); + moved.reset(); + EXPECT_EQ(destroyed, 1); +} + +struct SelfObserved { + LocalWeakPtr self; + bool& destroyed; + explicit SelfObserved(bool& destroyed) : destroyed(destroyed) {} + ~SelfObserved() { + EXPECT_TRUE(self.expired()); + EXPECT_FALSE(self.lock()); + // The implicit weak reference must protect the control while destruction + // drops the last explicit weak reference. + self.reset(); + destroyed = true; + } +}; +TEST(LocalSharedPtr, SelfWeakReferenceCanReleaseDuringDestruction) { + bool destroyed = false; + auto state = MakeLocalShared(destroyed); + state->self = state; + state.reset(); + EXPECT_TRUE(destroyed); +} + +struct Incomplete; +struct OpaqueHandle { + LocalSharedPtr strong; + LocalWeakPtr weak; + ~OpaqueHandle() = default; +}; +// Instantiate destruction while the pointee is still incomplete, as in the +// private worker facade and public RC handles. +void DestroyOpaque(OpaqueHandle* handle) { delete handle; } +struct Incomplete { + int& destroyed; + explicit Incomplete(int& destroyed) : destroyed(destroyed) {} + ~Incomplete() { ++destroyed; } +}; +TEST(LocalSharedPtr, IncompletePointeeDestruction) { + int destroyed = 0; + auto* handle = new OpaqueHandle; + handle->strong = MakeLocalShared(destroyed); + handle->weak = handle->strong; + DestroyOpaque(handle); + EXPECT_EQ(destroyed, 1); +} + +struct Throwing { + Tracked member; + explicit Throwing(int& destroyed) : member(destroyed) { + throw std::runtime_error("constructor failed"); + } +}; +TEST(LocalSharedPtr, ConstructionFailureUnwindsMembers) { + int destroyed = 0; + EXPECT_THROW(MakeLocalShared(destroyed), std::runtime_error); + EXPECT_EQ(destroyed, 1); +} + +#ifndef NDEBUG +TEST(LocalSharedPtrDeathTest, ForeignThreadCopyAndReleaseAreRejected) { + auto state = MakeLocalShared(7); + EXPECT_DEATH( + { + std::thread other([&] { auto copy = state; }); + other.join(); + }, + "outside its owner thread"); + EXPECT_DEATH( + { + std::thread other([&] { state.reset(); }); + other.join(); + }, + "outside its owner thread"); +} +TEST(LocalSharedPtrDeathTest, ForeignThreadWeakLockAndReleaseAreRejected) { + auto state = MakeLocalShared(7); + LocalWeakPtr weak = state; + EXPECT_DEATH( + { + std::thread other([&] { auto retained = weak.lock(); }); + other.join(); + }, + "outside its owner thread"); + state.reset(); + EXPECT_DEATH( + { + std::thread other([&] { weak.reset(); }); + other.join(); + }, + "outside its owner thread"); +} +#endif +} // namespace +} // namespace bycorf diff --git a/tests/rdma_rc_test.cpp b/tests/rdma_rc_test.cpp index b117293..5232a97 100644 --- a/tests/rdma_rc_test.cpp +++ b/tests/rdma_rc_test.cpp @@ -6,13 +6,17 @@ #include #include +#include #include #include +#include +#include #include "../src/rdma/rc_internal.h" #include "bycorf/rdma/rc.h" #include "bycorf/runtime/worker.h" #ifdef BYCORF_RDMA_FAKE +#include "bycorf/net/server.h" #include "rdma_fake.h" #define RcLoop RcLoopFake #endif @@ -20,8 +24,8 @@ namespace bycorf::rdma::detail { class RcTestPeer { public: - static std::shared_ptr State(RcConnection& c) { return c.state_; } - static std::shared_ptr State(RcListener& l) { return l.state_; } + static LocalSharedPtr State(RcConnection& c) { return c.state_; } + static LocalSharedPtr State(RcListener& l) { return l.state_; } }; } // namespace bycorf::rdma::detail namespace { @@ -250,6 +254,44 @@ TEST_F(RcLoop, InvalidOptionsAndPrivateDataAreRejected) { } #ifdef BYCORF_RDMA_FAKE +TEST(RcThreadOwnership, ServerShutdownReclaimsSuspendedAcceptOnOwnerThread) { + rdma_fake::Reset(); + class ListeningService final : public Service { + public: + std::atomic entered{false}; + void Prepare(unsigned) override {} + void Stop() noexcept override {} + Task Run(Worker& worker, ServiceContext) override { + auto listener = RcListener::Listen(worker, "192.0.2.1", 0); + if (!listener.ok()) co_return listener.status(); + entered.store(true, std::memory_order_release); + // Shutdown destroys this suspended frame and its local listener on the + // worker, before the control thread destroys Server and Worker objects. + auto accepted = co_await listener->Accept(); + co_return accepted.status(); + } + } service; + { + Server server; + server.AddService(&service); + ASSERT_TRUE(server + .Start({.thread_count_ = 1, + .pin_workers_ = false, + .recv_buffer_count_ = 0}) + .ok()); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!service.entered.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + EXPECT_TRUE(service.entered.load(std::memory_order_acquire)); + server.RequestStop(); + server.WaitUntilStopped(); + EXPECT_EQ(server.exit_code(), 0); + } + rdma_fake::CheckEmpty(); +} + TEST_F(RcLoop, QueuedAcceptPreservesOrderAndRetainsFailedResults) { RcOptions options; options.send_wr = options.recv_wr = 4; @@ -258,7 +300,7 @@ TEST_F(RcLoop, QueuedAcceptPreservesOrderAndRetainsFailedResults) { auto state = Peer::State(*listener); auto engine = state->engine.lock(); std::vector clients; - std::vector> queued; + std::vector> queued; // Queue a burst before Accept so both the child table and FIFO must grow. for (unsigned i = 0; i < 12; ++i) { const std::array data{static_cast(i)}; @@ -299,6 +341,17 @@ TEST_F(RcLoop, QueuedAcceptPreservesOrderAndRetainsFailedResults) { for (const auto& connection : accepted) EXPECT_TRUE(connection.IsOpen()); } +TEST_F(RcLoop, LazyAcceptRetainsStateAfterListenerHandleDestruction) { + auto listener = RcListener::Listen(worker, ip, 0); + ASSERT_TRUE(listener.ok()); + LocalWeakPtr state = Peer::State(*listener); + auto accepting = listener->Accept(); + *listener = RcListener{}; + EXPECT_FALSE(state.expired()); + EXPECT_TRUE(absl::IsCancelled(Run(std::move(accepting)).status())); + EXPECT_TRUE(state.expired()); +} + TEST_F(RcLoop, ListenerCloseReleasesUnacceptedReadyStates) { auto listener = RcListener::Listen(worker, ip, 0); ASSERT_TRUE(listener.ok()); @@ -306,7 +359,7 @@ TEST_F(RcLoop, ListenerCloseReleasesUnacceptedReadyStates) { auto client = Run(RcConnection::Connect(worker, ip, listener->port())); ASSERT_TRUE(client.ok()) << client.status(); Until([&] { return !state->ready.empty(); }); - std::weak_ptr queued = + LocalWeakPtr queued = state->children.at(state->ready.front()); EXPECT_TRUE(Run(listener->Close()).ok()); EXPECT_TRUE(state->ready.empty()); From 468ee767f0643178b6914a5ec1c88f37f4f839fd Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 17 Nov 2026 13:25:05 +0000 Subject: [PATCH 4/5] add missed unlikely --- src/rdma/rc_engine.cpp | 5 +++-- src/rdma/rc_internal.h | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp index df202a7..13e30f3 100644 --- a/src/rdma/rc_engine.cpp +++ b/src/rdma/rc_engine.cpp @@ -509,11 +509,12 @@ void Engine::DrainCq(Device& device) { ibv_wc completions[32]; for (;;) { const int count = ibv_poll_cq(device.cq, 32, completions); - if (count < 0) { + if (count < 0) [[unlikely]] { FailDevice(device, absl::UnavailableError("ibv_poll_cq failed")); return; } - if (!count) return; + if (!count) [[unlikely]] + return; for (int i = 0; i < count; ++i) { const auto& wc = completions[i]; // No public data-posting API exists yet. Error completions still diff --git a/src/rdma/rc_internal.h b/src/rdma/rc_internal.h index 7d19597..4736b65 100644 --- a/src/rdma/rc_internal.h +++ b/src/rdma/rc_internal.h @@ -8,7 +8,6 @@ #include #include -#include #include #include "absl/container/flat_hash_map.h" @@ -145,9 +144,9 @@ class Engine { absl::Status error; rdma_event_channel* cm = nullptr; Poll cm_poll; - std::unordered_map> connections; - std::unordered_map> listeners; - std::unordered_map> devices; + absl::flat_hash_map> connections; + absl::flat_hash_map> listeners; + absl::flat_hash_map> devices; private: absl::Status StartTimer(const LocalSharedPtr& connection); From 325cc3c77deb09e73fc193380fa1a0c5092bbad3 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 17 Nov 2026 13:31:31 +0000 Subject: [PATCH 5/5] perf(rdma): mark event-loop failure paths as unlikely --- src/rdma/rc_engine.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/rdma/rc_engine.cpp b/src/rdma/rc_engine.cpp index 13e30f3..ace942f 100644 --- a/src/rdma/rc_engine.cpp +++ b/src/rdma/rc_engine.cpp @@ -479,7 +479,9 @@ void Engine::DrainCm() { rdma_cm_event* event = nullptr; if (rdma_get_cm_event(cm, &event)) { if (errno == EINTR) continue; - if (!Again()) FailAll(ErrnoStatus("rdma_get_cm_event")); + if (!Again()) [[unlikely]] { + FailAll(ErrnoStatus("rdma_get_cm_event")); + } return; } auto* id = event->id; @@ -513,8 +515,8 @@ void Engine::DrainCq(Device& device) { FailDevice(device, absl::UnavailableError("ibv_poll_cq failed")); return; } - if (!count) [[unlikely]] - return; + // Empty is the normal end of a drain, including notification rechecks. + if (!count) return; for (int i = 0; i < count; ++i) { const auto& wc = completions[i]; // No public data-posting API exists yet. Error completions still @@ -537,19 +539,20 @@ void Engine::DrainNotifications(Device& device, bool rearm) { void* context = nullptr; if (ibv_get_cq_event(device.channel, &cq, &context)) { if (errno == EINTR) continue; - if (!Again() && rearm) + if (!Again() && rearm) [[unlikely]] { FailDevice(device, ErrnoStatus("ibv_get_cq_event")); + } return; } ibv_ack_cq_events(cq, 1); - if (cq != device.cq) { + if (cq != device.cq) [[unlikely]] { FailDevice(device, absl::InternalError( "unexpected CQ on worker completion channel")); continue; } if (rearm && device.error.ok()) { const int result = ibv_req_notify_cq(cq, 0); - if (result) { + if (result) [[unlikely]] { FailDevice(device, absl::ErrnoToStatus(result, "ibv_req_notify_cq")); continue; } @@ -565,7 +568,9 @@ void Engine::DrainAsync(Device& device) { ibv_async_event event{}; if (ibv_get_async_event(device.context.get(), &event)) { if (errno == EINTR) continue; - if (!Again()) FailDevice(device, ErrnoStatus("ibv_get_async_event")); + if (!Again()) [[unlikely]] { + FailDevice(device, ErrnoStatus("ibv_get_async_event")); + } return; } const auto type = event.event_type; @@ -588,7 +593,7 @@ void Engine::DrainAsync(Device& device) { } } void Engine::OnPoll(Poll& poll, int result) { - if (result < 0 || (result & (POLLERR | POLLHUP | POLLNVAL))) { + if (result < 0 || (result & (POLLERR | POLLHUP | POLLNVAL))) [[unlikely]] { auto error = absl::UnavailableError(absl::StrCat("RDMA fd poll failed: ", result)); if (poll.device) @@ -609,7 +614,7 @@ void Engine::OnPoll(Poll& poll, int result) { break; } auto status = poll.Arm(); - if (!status.ok()) { + if (!status.ok()) [[unlikely]] { if (poll.device) FailDevice(*poll.device, status); else