From 12bfd3e31ed24930b9e81f8e0ef3fbe06f9d75c3 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 11 Nov 2026 05:58:01 +0000 Subject: [PATCH] feat(rdma): add worker facade and registered memory ownership --- CMakeLists.txt | 29 ++- README.md | 41 ++++ docs/architecture/README.md | 4 +- docs/architecture/io.md | 12 +- docs/architecture/rdma.md | 70 +++++++ include/bycorf/io/rdma_backend.h | 75 ++++++++ include/bycorf/io/rdma_memory.h | 160 ++++++++++++++++ include/bycorf/runtime/worker.h | 19 ++ src/io/rdma_memory.cpp | 245 ++++++++++++++++++++++++ src/runtime/worker.cpp | 5 + tests/rdma_memory_test.cpp | 315 +++++++++++++++++++++++++++++++ tests/rdma_registration_test.cpp | 113 +++++++++++ tests/worker_wait_test.cpp | 39 ++++ 13 files changed, 1121 insertions(+), 6 deletions(-) create mode 100644 docs/architecture/rdma.md create mode 100644 include/bycorf/io/rdma_backend.h create mode 100644 include/bycorf/io/rdma_memory.h create mode 100644 src/io/rdma_memory.cpp create mode 100644 tests/rdma_memory_test.cpp create mode 100644 tests/rdma_registration_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bad842d..97f9cdb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ option(BYCORF_ENABLE_WERROR "Treat warnings as errors" OFF) option(BYCORF_KERNEL_BYPASS "Include DPDK networking and SPDK storage (selected independently at startup)" OFF) +option(BYCORF_ENABLE_RDMA "Include RDMA facade and memory registration" OFF) option(BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE "Enable detailed cross-core queue and wake latency tracing" OFF) @@ -174,11 +175,22 @@ add_library(bycorf target_compile_definitions(bycorf PUBLIC # Public headers and consumers must agree on the backend-dependent layouts. - # Export one numeric switch for both capabilities, including zero when off. + # Export numeric capability switches, including zero when disabled. BYCORF_KERNEL_BYPASS=$ + BYCORF_ENABLE_RDMA=$ BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE=$ ) +# RDMA is an independent capability: do not discover its libraries when off. +if (BYCORF_ENABLE_RDMA) + find_package(PkgConfig REQUIRED) + pkg_check_modules(BYCORF_IBVERBS REQUIRED IMPORTED_TARGET libibverbs) + pkg_check_modules(BYCORF_RDMACM REQUIRED IMPORTED_TARGET librdmacm) + target_sources(bycorf PRIVATE src/io/rdma_memory.cpp) + target_link_libraries(bycorf PUBLIC + PkgConfig::BYCORF_IBVERBS PkgConfig::BYCORF_RDMACM) +endif() + if (BYCORF_KERNEL_BYPASS) include(cmake/Dpdk.cmake) set(BYCORF_SPDK_SOURCE_DIR @@ -465,6 +477,21 @@ if (BYCORF_BUILD_TESTS) target_link_libraries(bycorf_unit_tests PRIVATE bycorf::core GTest::gtest_main) bycorf_discover_tests(bycorf_unit_tests 30 "runtime;unit") + if (BYCORF_ENABLE_RDMA) + add_executable(bycorf_rdma_memory_tests tests/rdma_memory_test.cpp) + target_link_libraries(bycorf_rdma_memory_tests PRIVATE bycorf::core GTest::gtest_main) + foreach(symbol IN ITEMS ibv_get_device_list ibv_free_device_list + ibv_get_device_name ibv_open_device ibv_close_device ibv_alloc_pd + ibv_dealloc_pd ibv_reg_mr ibv_reg_mr_iova2 ibv_dereg_mr) + target_link_options(bycorf_rdma_memory_tests PRIVATE "LINKER:--wrap=${symbol}") + endforeach() + bycorf_discover_tests(bycorf_rdma_memory_tests 30 "runtime;rdma;unit") + + add_executable(bycorf_rdma_registration_tests tests/rdma_registration_test.cpp) + target_link_libraries(bycorf_rdma_registration_tests PRIVATE bycorf::core GTest::gtest_main) + bycorf_discover_tests(bycorf_rdma_registration_tests 30 "runtime;rdma;integration") + endif() + add_executable(bycorf_dpdk_memory_budget_tests tests/dpdk_memory_budget_test.cpp src/io/dpdk_memory_budget.cpp) target_link_libraries(bycorf_dpdk_memory_budget_tests PRIVATE diff --git a/README.md b/README.md index ed89e60..3deb16e 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ connections, and asynchronous I/O; applications provide services above - [Name](#name) - [Build and test](#build-and-test) +- [Optional RDMA registration](#optional-rdma-registration) - [Run the examples](#run-the-examples) - [Use Bycorf in a CMake project](#use-bycorf-in-a-cmake-project) - [Development](#development) @@ -178,6 +179,46 @@ Follow the [DPDK/SPDK build and test guide](docs/dpdk-prototype.md) for its additional dependencies, virtual-device checks, physical-device configuration, and experimental compatibility limits. +## Optional RDMA registration + +`BYCORF_ENABLE_RDMA` enables the private worker facade and explicit memory +registration API. It defaults to off and is independent of +`BYCORF_KERNEL_BYPASS`; enabling it requires libibverbs and librdmacm development +packages, without DPDK/SPDK. On Ubuntu: + +```bash +sudo apt-get install --no-install-recommends libibverbs-dev librdmacm-dev +cmake -S . -B build-rdma -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBYCORF_ENABLE_RDMA=ON +cmake --build build-rdma --parallel +ctest --test-dir build-rdma -R '^bycorf_RdmaMemory\.' --output-on-failure +``` + +The device-free `RdmaMemory` suite injects verbs failures and verifies lazy +initialization, cleanup order, range/permission checks, worker/context identity, +and retained registration lifetimes. The `RdmaRegistration` test skips unless +an explicit device is supplied. With an existing Soft-RoCE device (substitute +its name from `rdma link show`): + +```bash +BYCORF_RDMA_TEST_DEVICE=rxe0 ctest --test-dir build-rdma \ + -R '^bycorf_RdmaRegistration\.' --output-on-failure +``` + +This opens the selected device, registers and deregisters memory, creates real +QPs sharing a PD, and checks permissions, bounds, and mismatched PD rejection. +It requires access to the device's uverbs node and sufficient memlock. It does +not test a connected data-transfer or immediate-data protocol. No device is +opened by unused workers or test discovery. + +Use `Worker::OpenRdmaDevice(name)` or `AcquireRdmaDomain(owning_context)` on the +owner worker, then `RegisterRdmaMemory(domain, backing_span, explicit_access)`. +Keep the returned domain through QP destruction. `RegisteredMemory::Slice` +retains registration; `slice.Acquire(qp, local_access)` validates context/PD and +returns an SGE lease to retain through terminal completion. The caller owns +backing memory and must stop remote access before releasing registrations and +freeing it. See [RDMA ownership](docs/architecture/rdma.md) for remote descriptors, +permissions, CM context ownership, and the independent storage-buffer contract. + ## Run the examples The default build enables `BYCORF_BUILD_EXAMPLES`. If it was disabled, set diff --git a/docs/architecture/README.md b/docs/architecture/README.md index ec2f8bb..874c894 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -23,7 +23,7 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging. | Module | Responsibility | Document | |---|---|---| | Runtime | Worker lifecycle, coroutine scheduling, sleep coordination, cross-worker and foreign ingress | [Runtime](runtime.md) | -| I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) | +| I/O | Worker-owned shared io_uring, network/storage adapters, optional RDMA registration and SPDK storage, shared DPDK environment | [I/O backends](io.md), [RDMA ownership](rdma.md) | | Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) | | RPC | Framed request/reply protocol above streams | [RPC](rpc.md) | @@ -38,6 +38,8 @@ worker before that coroutine uses its stream again. Startup selection independently activates network and storage backends and remains fixed for the process lifetime. Both default to io_uring even in a bypass-capable build. +`BYCORF_ENABLE_RDMA` independently enables the RDMA facade and registration +layer; resources are acquired only on explicit use. The [prototype runbook](../dpdk-prototype.md) describes the DPDK configuration. Architecture documents describe the current core model and stable tradeoffs. diff --git a/docs/architecture/io.md b/docs/architecture/io.md index 9ad7fb5..a4a317a 100644 --- a/docs/architecture/io.md +++ b/docs/architecture/io.md @@ -18,8 +18,10 @@ limitations under the License. `Worker` owns one `IoUringBackend` for kernel sockets, file I/O, completion delivery, timers, and MSG_RING/eventfd wakeups. Its `NetBackend` and -`StorageBackend` facades share that ring. Network receive buffers and -registered storage buffers have separate lifecycles. Linux TCP uses +`StorageBackend` facades share that ring. The independently optional +`RdmaBackend` also uses this ring for channel readiness. Network receive +buffers, registered storage buffers and RDMA registrations have separate +lifecycles; see [RDMA ownership](rdma.md). Linux TCP uses provided-buffer multishot receives with a per-connection one-shot fallback. `BYCORF_KERNEL_BYPASS` is the single build option for DPDK networking and SPDK @@ -42,8 +44,10 @@ Network adapters supply notification preparation, idle checks, deadlines, and cleanup to the [worker sleep protocol](runtime.md#sleep-and-wakeups). The worker enters the ring's blocking wait; adapters never block independently. -The ring outlives both adapters. The worker initializes it first, stops storage -and networking in that order, then shuts it down. Storage dispatch remains +The ring outlives all facades. RDMA drops its weak resource cache on shutdown; +registration and operation handles retain their PD/context independently. The +worker initializes the ring first, stops storage and networking in that order, +then shuts the ring down. Storage dispatch remains valid through shutdown completion draining; SPDK callbacks use the owning worker's TLS to reach its storage adapter. Kernel registrations remain live until ring shutdown quiesces kernel access. diff --git a/docs/architecture/rdma.md b/docs/architecture/rdma.md new file mode 100644 index 0000000..0f17263 --- /dev/null +++ b/docs/architecture/rdma.md @@ -0,0 +1,70 @@ + + +# RDMA registration and ownership + +`BYCORF_ENABLE_RDMA` adds an independent I/O capability backed by libibverbs and +librdmacm. It defaults to off and does not select or require DPDK/SPDK. The +private `RdmaBackend` facade reaches the worker's existing io_uring +through `KernelBackend()` for channel readiness and cancellation. It creates no +ring, polling thread, thread pool, or process-wide blocking lock. The worker +continues to own shared-ring submission and completion dispatch. + +Construction does not enumerate or open devices. Explicit `OpenRdmaDevice` +selects a named verbs device; `AcquireRdmaDomain` accepts an owning reference to +the actual context selected by the caller, including a CM-selected context. +An aliasing shared pointer can keep the CM context owner alive. A borrowed raw +context with a no-op deleter is insufficient. Resource acquisition and cache +access occur on the owning worker. Each worker caches live PDs by actual +context identity; two contexts for one device and two workers using one context +have distinct PDs. The cache holds weak references, so resource ownership is +independent of worker shutdown. Partial initialization unwinds in reverse order. + +`RdmaProtectionDomain` retains its PD and context. QP owners create QPs in that +PD and retain the domain until QP destruction. A registered MR can be used by +multiple QPs in the same PD; it belongs to neither a particular QP nor an IMM +(immediate-data) operation. Operation leases check both the QP's context and PD. +Connection management, QP posting and completion protocols are outside this +registration layer. + +`RegisteredMemory` registers caller-owned backing memory with explicit access +permissions. Copies, registered slices, and operation leases retain the same +MR. The last reference deregisters it before releasing its PD and context. +An operation keeps its lease until terminal completion, including error and +cancellation; copying the SGE alone does not retain registration. The application +keeps backing memory alive through all local completions and remote quiescence, +then releases every registration reference before freeing the memory. Dropping +a registration handle or shutting down a worker does not stop remote access. +Teardown failures terminate rather than release dependencies still potentially +used by hardware. + +Slices validate nonempty ranges without arithmetic overflow. Local reads need +no access flag; local DMA writes require local-write permission. Remote write +and atomic registration also require local-write permission. Unsupported access +bits are rejected. `RemoteMemoryRegion{address, length, rkey}` is a non-owning +peer descriptor with checked subranges, not a prescribed wire encoding. Export +requires explicit granted remote permissions. Its length is an application +contract: a slice does not narrow the hardware rkey's authority over the MR, +and exporting a subset of permissions does not reduce the MR's granted rights. + +RDMA registration does not reuse storage `RegisterBuffers`, allocate backing +memory, grow arenas, or supply a business buffer pool. Memory used by both +storage and RDMA must independently satisfy both registrations' alignment, +access and lifetime requirements. + +Sources: `include/bycorf/io/rdma_backend.h`, +`include/bycorf/io/rdma_memory.h`, `src/io/rdma_memory.cpp`, +`include/bycorf/runtime/worker.h`, `src/runtime/worker.cpp`. diff --git a/include/bycorf/io/rdma_backend.h b/include/bycorf/io/rdma_backend.h new file mode 100644 index 0000000..cf5519f --- /dev/null +++ b/include/bycorf/io/rdma_backend.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BYCORF_IO_RDMA_BACKEND_H_ +#define BYCORF_IO_RDMA_BACKEND_H_ + +#if BYCORF_ENABLE_RDMA +#include "bycorf/io/io_uring_backend.h" +#include "bycorf/io/rdma_memory.h" + +namespace bycorf { +// Private CRTP facade: the owner supplies its existing KernelBackend(). Neither +// construction nor destruction downcasts or accesses devices/the shared ring. +// All calls run on the owning worker; external handles can outlive the facade. +template +class RdmaBackend { + public: + // Explicit device selection; no default-device fallback. Opens lazily and + // reuses a live worker-local PD for this context/device. + absl::StatusOr OpenRdmaDevice(std::string_view name) { + return resources_.OpenDevice(name); + } + // For a CM-selected context, pass an owning shared_ptr (an aliasing pointer + // may retain its CM owner). It must keep the exact context alive; wrapping a + // borrowed pointer with a no-op deleter does not satisfy this contract. + absl::StatusOr AcquireRdmaDomain( + const std::shared_ptr& context) { + return resources_.Domain(context); + } + // Rejects domains from another worker, invalid ranges and invalid access. + // Caller owns memory; this API neither allocates nor grows backing buffers. + absl::StatusOr RegisterRdmaMemory( + const RdmaProtectionDomain& domain, std::span memory, + RdmaAccess access) { + return resources_.Register(domain, memory, access); + } + // Channel readiness uses the owner's single ring. The caller retains fd/tag + // through terminal completion, including cancellation, as for kernel I/O. + absl::Status SubmitRdmaEventPoll(int fd, unsigned events, IoCompletion* tag) { + return Kernel().SubmitPoll(fd, events, tag); + } + absl::Status CancelRdmaEventPoll(IoCompletion* tag) { + return Kernel().SubmitCancel(tag); + } + // Drops only cache entries. Outstanding handles continue to own resources; + // callers must quiesce QPs/remote access before releasing those handles. + void Shutdown() noexcept { resources_.Shutdown(); } + + private: + friend Owner; + RdmaBackend() = default; + ~RdmaBackend() = default; + RdmaBackend(const RdmaBackend&) = delete; + RdmaBackend& operator=(const RdmaBackend&) = delete; + IoUringBackend& Kernel() noexcept { + return static_cast(*this).KernelBackend(); + } + detail::RdmaResources resources_; +}; +} // namespace bycorf +#endif // BYCORF_ENABLE_RDMA +#endif // BYCORF_IO_RDMA_BACKEND_H_ diff --git a/include/bycorf/io/rdma_memory.h b/include/bycorf/io/rdma_memory.h new file mode 100644 index 0000000..f68f0ff --- /dev/null +++ b/include/bycorf/io/rdma_memory.h @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BYCORF_IO_RDMA_MEMORY_H_ +#define BYCORF_IO_RDMA_MEMORY_H_ + +#if BYCORF_ENABLE_RDMA +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/status/statusor.h" + +namespace bycorf { + +// Explicit registration permissions. Local reads are always allowed by verbs. +// Remote write/atomic require kLocalWrite; unsupported/unknown bits are +// rejected. +enum class RdmaAccess : unsigned { + kNone = 0, + kLocalWrite = IBV_ACCESS_LOCAL_WRITE, + kRemoteWrite = IBV_ACCESS_REMOTE_WRITE, + kRemoteRead = IBV_ACCESS_REMOTE_READ, + kRemoteAtomic = IBV_ACCESS_REMOTE_ATOMIC, +}; +constexpr RdmaAccess operator|(RdmaAccess a, RdmaAccess b) noexcept { + return static_cast(static_cast(a) | + static_cast(b)); +} + +// A peer descriptor, not an owning handle or a wire serialization format. +// The application exchanges it and revokes remote access before deregistration. +// A sliced descriptor narrows our range checks, not the hardware rkey's +// authority. +struct RemoteMemoryRegion { + std::uint64_t address = 0; + std::uint64_t length = 0; + std::uint32_t rkey = 0; + + // Rejects empty, overflowing, or out-of-bounds subranges. + absl::StatusOr Slice(std::uint64_t offset, + std::uint64_t size) const; +}; + +namespace detail { +struct RdmaDomainState; +struct RdmaRegistration; +class RdmaResources; +} // namespace detail + +// Retains a PD and its actual verbs context. QP owners must retain this handle +// until after destroying their QPs. Multiple QPs in this PD may use every MR +// registered in it; an MR is not owned by a QP or an immediate-data operation. +class RdmaProtectionDomain { + public: + RdmaProtectionDomain() = default; + // Borrowed for QP construction; never deallocate it outside this owner. + ibv_pd* native_pd() const noexcept; + explicit operator bool() const noexcept { return state_ != nullptr; } + + private: + friend class detail::RdmaResources; + std::shared_ptr state_; +}; + +enum class RdmaLocalAccess { kRead, kWrite }; + +// An operation retains this lease until terminal completion (including error +// and cancellation). It owns registration, PD and context, NEVER backing +// memory. The QP's separate owner must also survive completion. Copying the SGE +// alone does not retain any resources. +class RdmaMemoryLease { + public: + const ibv_sge& sge() const noexcept { return sge_; } + + private: + friend class RegisteredMemorySlice; + std::shared_ptr registration_; + ibv_sge sge_{}; +}; + +// Copyable registration-retaining view; slices do not register another MR. +class RegisteredMemorySlice { + public: + RegisteredMemorySlice() = default; + std::size_t size() const noexcept { return registration_ ? length_ : 0; } + absl::StatusOr Slice(std::size_t offset, + std::size_t length) const; + // Checks actual QP context/PD, local permission and the 32-bit SGE length. + absl::StatusOr Acquire(const ibv_qp& qp, + RdmaLocalAccess access) const; + // Requires a nonempty subset of granted remote permissions. The returned + // descriptor does not retain this registration or enforce peer cooperation. + absl::StatusOr RemoteRegion(RdmaAccess access) const; + + private: + friend class RegisteredMemory; + std::shared_ptr registration_; + std::size_t offset_ = 0; + std::size_t length_ = 0; +}; + +// Owns only the verbs registration. Copies, slices and operation leases retain +// it; the last reference deregisters the MR before releasing PD/context. +// Caller keeps the original allocation alive through all local completions AND +// remote quiescence, and releases all registration references before freeing +// it. Storage RegisterBuffers is an independent registration/lifetime contract. +class RegisteredMemory { + public: + RegisteredMemory() = default; + std::size_t size() const noexcept; + RdmaAccess access() const noexcept; + absl::StatusOr Slice(std::size_t offset, + std::size_t length) const; + + private: + friend class detail::RdmaResources; + std::shared_ptr registration_; +}; + +namespace detail { +// Worker-local weak cache. No allocation/device access until explicit use; +// handles, not the worker cache, determine resource lifetimes. Calls are +// confined to the owner worker. No global registries, blocking locks or memory +// pools. +class RdmaResources { + public: + absl::StatusOr OpenDevice(std::string_view name); + absl::StatusOr Domain( + const std::shared_ptr& context); + absl::StatusOr Register(const RdmaProtectionDomain& domain, + std::span memory, + RdmaAccess access); + void Shutdown() noexcept { domains_.clear(); } + + private: + std::vector> domains_; +}; +} // namespace detail +} // namespace bycorf +#endif // BYCORF_ENABLE_RDMA +#endif // BYCORF_IO_RDMA_MEMORY_H_ diff --git a/include/bycorf/runtime/worker.h b/include/bycorf/runtime/worker.h index 2be3f80..3fe2512 100644 --- a/include/bycorf/runtime/worker.h +++ b/include/bycorf/runtime/worker.h @@ -34,6 +34,7 @@ #include "bycorf/base/inline_ring_queue.h" #include "bycorf/io/completion.h" #include "bycorf/io/net_backend.h" +#include "bycorf/io/rdma_backend.h" #include "bycorf/io/storage.h" #include "bycorf/io/storage_backend.h" #include "bycorf/net/connection.h" @@ -72,11 +73,26 @@ struct WorkerOptions { // Shutdown drains them while the complete worker and its queues are alive. class Worker : private IoUringBackend, private NetBackend, +#if BYCORF_ENABLE_RDMA + private RdmaBackend, +#endif private StorageBackend { +#if BYCORF_ENABLE_RDMA + using Rdma = RdmaBackend; +#endif using Network = NetBackend; using Storage = StorageBackend; public: +#if BYCORF_ENABLE_RDMA + // RDMA resources are acquired explicitly on this worker, independently of + // network/storage backend selection. See rdma_memory.h for buffer lifetimes. + using Rdma::AcquireRdmaDomain; + using Rdma::CancelRdmaEventPoll; + using Rdma::OpenRdmaDevice; + using Rdma::RegisterRdmaMemory; + using Rdma::SubmitRdmaEventPoll; +#endif struct WakeStats { std::uint64_t checks_ = 0; std::uint64_t sent_ = 0; @@ -368,6 +384,9 @@ class Worker : private IoUringBackend, private: friend class NetBackend; friend class StorageBackend; +#if BYCORF_ENABLE_RDMA + friend class RdmaBackend; +#endif // Shared-ring access for the CRTP facades; the base conversion is a fixed // address adjustment rather than a load of a stored pointer. IoUringBackend& KernelBackend() noexcept { return *this; } diff --git a/src/io/rdma_memory.cpp b/src/io/rdma_memory.cpp new file mode 100644 index 0000000..5913a94 --- /dev/null +++ b/src/io/rdma_memory.cpp @@ -0,0 +1,245 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bycorf/io/rdma_memory.h" + +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "spdlog/spdlog.h" + +namespace bycorf { +namespace { +constexpr unsigned kLocalWrite = static_cast(RdmaAccess::kLocalWrite); +constexpr unsigned kRemoteWrite = + static_cast(RdmaAccess::kRemoteWrite); +constexpr unsigned kRemoteRead = static_cast(RdmaAccess::kRemoteRead); +constexpr unsigned kRemoteAtomic = + static_cast(RdmaAccess::kRemoteAtomic); +constexpr unsigned kRemote = kRemoteWrite | kRemoteRead | kRemoteAtomic; + +absl::Status VerbsError(const char* operation) { + const int error = errno; + return absl::ErrnoToStatus(error, absl::StrCat(operation, " failed")); +} + +// A failed teardown may leave resources referenced by hardware. Continuing +// could release their PD/context too early. Destructors cannot report errors, +// so fail closed instead of hiding a lifetime violation or provider failure. +void CheckRelease(int result, const char* operation) noexcept { + if (result != 0) { + spdlog::critical("{} failed during RDMA resource teardown: {}", operation, + result); + std::terminate(); + } +} + +absl::Status ValidateRange(std::size_t total, std::size_t offset, + std::size_t length) { + if (length == 0 || offset > total || length > total - offset) { + return absl::InvalidArgumentError("empty or out-of-bounds RDMA range"); + } + return absl::OkStatus(); +} +} // namespace + +namespace detail { +struct RdmaDomainState { + std::shared_ptr context; + ibv_pd* pd = nullptr; + ~RdmaDomainState() { + if (pd) CheckRelease(ibv_dealloc_pd(pd), "ibv_dealloc_pd"); + } +}; + +struct RdmaRegistration { + std::shared_ptr domain; + ibv_mr* mr = nullptr; + RdmaAccess access = RdmaAccess::kNone; + ~RdmaRegistration() { + if (mr) CheckRelease(ibv_dereg_mr(mr), "ibv_dereg_mr"); + } +}; + +absl::StatusOr RdmaResources::Domain( + const std::shared_ptr& context) { + if (!context || context.use_count() == 0) { + return absl::InvalidArgumentError("RDMA context must have an owner"); + } + std::erase_if(domains_, [](const auto& weak) { return weak.expired(); }); + for (const auto& weak : domains_) { + if (auto state = weak.lock(); + state && state->context.get() == context.get()) { + RdmaProtectionDomain result; + result.state_ = std::move(state); + return result; + } + } + // Allocate the C++ owner before acquiring verbs resources, including when + // cache growth later throws. Each failed attempt leaves no partial PD behind. + auto state = std::make_shared(); + state->context = context; + state->pd = ibv_alloc_pd(context.get()); + if (!state->pd) return VerbsError("ibv_alloc_pd"); + domains_.push_back(state); + RdmaProtectionDomain result; + result.state_ = std::move(state); + return result; +} + +absl::StatusOr RdmaResources::OpenDevice( + std::string_view name) { + if (name.empty()) + return absl::InvalidArgumentError("RDMA device name is empty"); + for (const auto& weak : domains_) { + if (auto state = weak.lock(); + state && name == ibv_get_device_name(state->context->device)) { + RdmaProtectionDomain result; + result.state_ = std::move(state); + return result; + } + } + int count = 0; + std::unique_ptr devices( + ibv_get_device_list(&count), ibv_free_device_list); + if (!devices) return VerbsError("ibv_get_device_list"); + for (int i = 0; i < count; ++i) { + if (name != ibv_get_device_name(devices.get()[i])) continue; + auto* opened = ibv_open_device(devices.get()[i]); + if (!opened) return VerbsError("ibv_open_device"); + // shared_ptr invokes this deleter even if allocating its control block + // throws, so a successful open never escapes without an owner. + std::shared_ptr context(opened, [](ibv_context* value) { + CheckRelease(ibv_close_device(value), "ibv_close_device"); + }); + return Domain(context); + } + return absl::NotFoundError(absl::StrCat("RDMA device not found: ", name)); +} + +absl::StatusOr RdmaResources::Register( + const RdmaProtectionDomain& domain, std::span memory, + RdmaAccess access) { + if (!domain || + !std::any_of(domains_.begin(), domains_.end(), [&](const auto& weak) { + return weak.lock() == domain.state_; + })) { + return absl::InvalidArgumentError("RDMA PD does not belong to this worker"); + } + const auto address = reinterpret_cast(memory.data()); + if (!memory.data() || memory.empty() || + memory.size() > std::numeric_limits::max() - address) { + return absl::InvalidArgumentError("empty or overflowing RDMA memory range"); + } + const auto flags = static_cast(access); + if ((flags & ~(kLocalWrite | kRemote)) != 0 || + ((flags & (kRemoteWrite | kRemoteAtomic)) && !(flags & kLocalWrite))) { + return absl::InvalidArgumentError("invalid RDMA registration permissions"); + } + auto registration = std::make_shared(); + registration->domain = domain.state_; + registration->access = access; + registration->mr = ibv_reg_mr(domain.native_pd(), memory.data(), + memory.size(), static_cast(flags)); + if (!registration->mr) return VerbsError("ibv_reg_mr"); + RegisteredMemory result; + result.registration_ = std::move(registration); + return result; +} +} // namespace detail + +ibv_pd* RdmaProtectionDomain::native_pd() const noexcept { + return state_ ? state_->pd : nullptr; +} + +std::size_t RegisteredMemory::size() const noexcept { + return registration_ ? registration_->mr->length : 0; +} +RdmaAccess RegisteredMemory::access() const noexcept { + return registration_ ? registration_->access : RdmaAccess::kNone; +} +absl::StatusOr RegisteredMemory::Slice( + std::size_t offset, std::size_t length) const { + auto status = ValidateRange(size(), offset, length); + if (!status.ok()) return status; + RegisteredMemorySlice result; + result.registration_ = registration_; + result.offset_ = offset; + result.length_ = length; + return result; +} +absl::StatusOr RegisteredMemorySlice::Slice( + std::size_t offset, std::size_t length) const { + auto status = ValidateRange(size(), offset, length); + if (!status.ok()) return status; + RegisteredMemorySlice result = *this; + result.offset_ += offset; + result.length_ = length; + return result; +} +absl::StatusOr RegisteredMemorySlice::Acquire( + const ibv_qp& qp, RdmaLocalAccess access) const { + if (!registration_) return absl::FailedPreconditionError("empty RDMA slice"); + const auto& domain = *registration_->domain; + if (qp.pd != domain.pd || qp.context != domain.context.get()) { + return absl::InvalidArgumentError( + "RDMA QP and MR must share context and PD"); + } + if (access != RdmaLocalAccess::kRead && access != RdmaLocalAccess::kWrite) { + return absl::InvalidArgumentError("invalid RDMA local access"); + } + if (access == RdmaLocalAccess::kWrite && + !(static_cast(registration_->access) & kLocalWrite)) { + return absl::PermissionDeniedError("MR does not allow local writes"); + } + if (length_ > std::numeric_limits::max()) { + return absl::InvalidArgumentError("RDMA slice exceeds one SGE's length"); + } + RdmaMemoryLease result; + result.registration_ = registration_; + result.sge_.addr = + reinterpret_cast(registration_->mr->addr) + offset_; + result.sge_.length = static_cast(length_); + result.sge_.lkey = registration_->mr->lkey; + return result; +} +absl::StatusOr RegisteredMemorySlice::RemoteRegion( + RdmaAccess access) const { + if (!registration_) return absl::FailedPreconditionError("empty RDMA slice"); + const auto requested = static_cast(access); + if (!requested || (requested & ~kRemote)) { + return absl::InvalidArgumentError("expected explicit remote permissions"); + } + if ((requested & static_cast(registration_->access)) != requested) { + return absl::PermissionDeniedError("MR lacks requested remote permissions"); + } + return RemoteMemoryRegion{ + reinterpret_cast(registration_->mr->addr) + offset_, + length_, registration_->mr->rkey}; +} +absl::StatusOr RemoteMemoryRegion::Slice( + std::uint64_t offset, std::uint64_t size) const { + if (!size || offset > length || size > length - offset || + length > std::numeric_limits::max() - address) { + return absl::InvalidArgumentError( + "empty, overflowing or out-of-bounds remote range"); + } + return RemoteMemoryRegion{address + offset, size, rkey}; +} +} // namespace bycorf diff --git a/src/runtime/worker.cpp b/src/runtime/worker.cpp index 4424743..5a82d84 100644 --- a/src/runtime/worker.cpp +++ b/src/runtime/worker.cpp @@ -236,6 +236,11 @@ absl::Status Worker::Init(const WorkerOptions& options) { } void Worker::Shutdown() { +#if BYCORF_ENABLE_RDMA + // Registration can be used before Init; its handles retain verbs resources + // independently of the ring. Never invalidate a live MR during shutdown. + Rdma::Shutdown(); +#endif if (!initialized_) { return; } diff --git a/tests/rdma_memory_test.cpp b/tests/rdma_memory_test.cpp new file mode 100644 index 0000000..07c67fe --- /dev/null +++ b/tests/rdma_memory_test.cpp @@ -0,0 +1,315 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "bycorf/runtime/worker.h" + +namespace { +struct VerbsState { + int lists = 0, opens = 0, pds = 0, mrs = 0; + int device_calls = 0; + int last_access = -1; + bool fail_list = false, fail_open = false, fail_pd = false, fail_mr = false; + std::vector releases; +} verbs; +ibv_device device{}; + +class RdmaMemory : public testing::Test { + protected: + void SetUp() override { verbs = {}; } + void TearDown() override { + EXPECT_EQ(verbs.lists, 0); + EXPECT_EQ(verbs.opens, 0); + EXPECT_EQ(verbs.pds, 0); + EXPECT_EQ(verbs.mrs, 0); + } +}; +} // namespace + +extern "C" { +ibv_device** __wrap_ibv_get_device_list(int* count) { + ++verbs.device_calls; + if (verbs.fail_list) { + errno = ENODEV; + return nullptr; + } + ++verbs.lists; + *count = 1; + return new ibv_device*[2]{&device, nullptr}; +} +void __wrap_ibv_free_device_list(ibv_device** devices) { + --verbs.lists; + delete[] devices; +} +const char* __wrap_ibv_get_device_name(ibv_device*) { return "test0"; } +ibv_context* __wrap_ibv_open_device(ibv_device* dev) { + ++verbs.device_calls; + if (verbs.fail_open) { + errno = EACCES; + return nullptr; + } + ++verbs.opens; + auto* context = new ibv_context{}; + context->device = dev; + return context; +} +int __wrap_ibv_close_device(ibv_context* context) { + --verbs.opens; + verbs.releases.push_back("context"); + delete context; + return 0; +} +ibv_pd* __wrap_ibv_alloc_pd(ibv_context* context) { + ++verbs.device_calls; + if (verbs.fail_pd) { + errno = ENOMEM; + return nullptr; + } + ++verbs.pds; + auto* pd = new ibv_pd{}; + pd->context = context; + return pd; +} +int __wrap_ibv_dealloc_pd(ibv_pd* pd) { + --verbs.pds; + verbs.releases.push_back("pd"); + delete pd; + return 0; +} +ibv_mr* __wrap_ibv_reg_mr(ibv_pd* pd, void* addr, size_t length, int access) { + if (verbs.fail_mr) { + errno = ENOMEM; + return nullptr; + } + ++verbs.mrs; + verbs.last_access = access; + auto* mr = new ibv_mr{}; + mr->context = pd->context; + mr->pd = pd; + mr->addr = addr; + mr->length = length; + mr->lkey = 17; + mr->rkey = 29; + return mr; +} +// verbs headers dispatch runtime access flags through iova2; optimized builds +// may use the older entry point. Intercept both without changing production. +ibv_mr* __wrap_ibv_reg_mr_iova2(ibv_pd* pd, void* addr, size_t length, + uint64_t iova, unsigned access) { + EXPECT_EQ(iova, reinterpret_cast(addr)); + return __wrap_ibv_reg_mr(pd, addr, length, static_cast(access)); +} +int __wrap_ibv_dereg_mr(ibv_mr* mr) { + --verbs.mrs; + verbs.releases.push_back("mr"); + delete mr; + return 0; +} +} // extern "C" + +namespace { +using bycorf::RdmaAccess; +using bycorf::RdmaLocalAccess; + +TEST_F(RdmaMemory, UnusedWorkerNeverAccessesDevices) { + verbs.fail_list = verbs.fail_open = verbs.fail_pd = true; + { + bycorf::Worker worker; + worker.Shutdown(); + } + EXPECT_TRUE(verbs.releases.empty()); + EXPECT_EQ(verbs.device_calls, 0); +} + +TEST_F(RdmaMemory, FailedInitializationReleasesPartialResourcesAndCanRetry) { + bycorf::Worker worker; + verbs.fail_list = true; + EXPECT_FALSE(worker.OpenRdmaDevice("test0").ok()); + verbs.fail_list = false; + verbs.fail_open = true; + EXPECT_FALSE(worker.OpenRdmaDevice("test0").ok()); + EXPECT_EQ(verbs.lists, 0); + verbs.fail_open = false; + verbs.fail_pd = true; + EXPECT_FALSE(worker.OpenRdmaDevice("test0").ok()); + EXPECT_EQ(verbs.opens, 0); + EXPECT_EQ(verbs.pds, 0); + verbs.fail_pd = false; + auto domain = worker.OpenRdmaDevice("test0"); + ASSERT_TRUE(domain.ok()) << domain.status(); + std::array memory; + verbs.fail_mr = true; + EXPECT_FALSE( + worker.RegisterRdmaMemory(*domain, memory, RdmaAccess::kNone).ok()); + EXPECT_EQ(verbs.mrs, 0); + EXPECT_EQ(verbs.pds, 1); + verbs.fail_mr = false; + EXPECT_TRUE( + worker.RegisterRdmaMemory(*domain, memory, RdmaAccess::kNone).ok()); + EXPECT_FALSE(worker.OpenRdmaDevice("missing").ok()); + EXPECT_FALSE(worker.OpenRdmaDevice("").ok()); + EXPECT_FALSE(worker.AcquireRdmaDomain({}).ok()); +} + +TEST_F(RdmaMemory, ContextIdentityAndWorkerOwnership) { + bycorf::Worker first, second; + auto a = first.OpenRdmaDevice("test0"); + auto b = first.OpenRdmaDevice("test0"); + ASSERT_TRUE(a.ok()); + ASSERT_TRUE(b.ok()); + EXPECT_EQ(a->native_pd(), b->native_pd()); + // Same device, distinct actual context: must not reuse the first PD. + auto context = std::shared_ptr(__wrap_ibv_open_device(&device), + __wrap_ibv_close_device); + auto c = first.AcquireRdmaDomain(context); + auto d = second.AcquireRdmaDomain(context); + ASSERT_TRUE(c.ok()); + ASSERT_TRUE(d.ok()); + EXPECT_NE(a->native_pd(), c->native_pd()); + EXPECT_NE(c->native_pd(), d->native_pd()); + EXPECT_EQ(c->native_pd()->context, d->native_pd()->context); + std::array memory; + EXPECT_FALSE(first.RegisterRdmaMemory(*d, memory, RdmaAccess::kNone).ok()); + EXPECT_FALSE(first.RegisterRdmaMemory({}, memory, RdmaAccess::kNone).ok()); +} + +TEST_F(RdmaMemory, LeaseKeepsRegistrationAndContextAfterWorkerDestruction) { + std::array backing; + bycorf::RdmaMemoryLease retained; + { + bycorf::Worker worker; + auto domain = worker.OpenRdmaDevice("test0"); + ASSERT_TRUE(domain.ok()); + auto mr = worker.RegisterRdmaMemory( + *domain, backing, RdmaAccess::kLocalWrite | RdmaAccess::kRemoteWrite); + ASSERT_TRUE(mr.ok()); + auto slice = mr->Slice(8, 24); + ASSERT_TRUE(slice.ok()); + auto sub = slice->Slice(4, 8); + ASSERT_TRUE(sub.ok()); + ibv_qp qp{}; + qp.pd = domain->native_pd(); + qp.context = qp.pd->context; + auto lease = sub->Acquire(qp, RdmaLocalAccess::kWrite); + ASSERT_TRUE(lease.ok()); + retained = *lease; + EXPECT_EQ(retained.sge().addr, + reinterpret_cast(backing.data() + 12)); + EXPECT_EQ(retained.sge().length, 8); + EXPECT_EQ(retained.sge().lkey, 17); + worker.Shutdown(); + EXPECT_EQ(verbs.mrs, 1); + } + EXPECT_EQ(verbs.mrs, 1); + EXPECT_EQ(verbs.pds, 1); + EXPECT_EQ(verbs.opens, 1); + retained = {}; + EXPECT_EQ(verbs.releases, (std::vector{"mr", "pd", "context"})); +} + +TEST_F(RdmaMemory, PermissionsRangesAndProtectionDomainChecks) { + std::array backing; + bycorf::Worker worker; + auto domain = worker.OpenRdmaDevice("test0"); + ASSERT_TRUE(domain.ok()); + EXPECT_FALSE(worker.RegisterRdmaMemory(*domain, {}, RdmaAccess::kNone).ok()); + for (auto invalid : {RdmaAccess::kRemoteWrite, RdmaAccess::kRemoteAtomic, + static_cast(1U << 30)}) { + EXPECT_FALSE(worker.RegisterRdmaMemory(*domain, backing, invalid).ok()); + } + auto mr = + worker.RegisterRdmaMemory(*domain, backing, RdmaAccess::kRemoteRead); + ASSERT_TRUE(mr.ok()); + EXPECT_EQ(verbs.last_access, IBV_ACCESS_REMOTE_READ); + EXPECT_EQ(mr->access(), RdmaAccess::kRemoteRead); + EXPECT_EQ(mr->size(), backing.size()); + EXPECT_FALSE(mr->Slice(0, 0).ok()); + EXPECT_FALSE(mr->Slice(64, 1).ok()); + EXPECT_FALSE(mr->Slice(1, std::numeric_limits::max()).ok()); + auto slice = mr->Slice(8, 32); + ASSERT_TRUE(slice.ok()); + EXPECT_FALSE(slice->Slice(31, 2).ok()); + ibv_qp qp{}; + qp.pd = domain->native_pd(); + qp.context = qp.pd->context; + EXPECT_TRUE(slice->Acquire(qp, RdmaLocalAccess::kRead).ok()); + EXPECT_TRUE(absl::IsPermissionDenied( + slice->Acquire(qp, RdmaLocalAccess::kWrite).status())); + ibv_pd wrong{}; + wrong.context = qp.context; + qp.pd = &wrong; + EXPECT_FALSE(slice->Acquire(qp, RdmaLocalAccess::kRead).ok()); + qp.pd = domain->native_pd(); + qp.context = nullptr; + EXPECT_FALSE(slice->Acquire(qp, RdmaLocalAccess::kRead).ok()); + auto remote = slice->RemoteRegion(RdmaAccess::kRemoteRead); + ASSERT_TRUE(remote.ok()); + EXPECT_EQ(remote->address, reinterpret_cast(backing.data() + 8)); + EXPECT_EQ(remote->length, 32); + EXPECT_EQ(remote->rkey, 29); + EXPECT_FALSE(slice->RemoteRegion(RdmaAccess::kNone).ok()); + EXPECT_FALSE(slice->RemoteRegion(RdmaAccess::kLocalWrite).ok()); + EXPECT_TRUE(absl::IsPermissionDenied( + slice->RemoteRegion(RdmaAccess::kRemoteWrite).status())); + EXPECT_FALSE(bycorf::RegisteredMemory{}.Slice(0, 1).ok()); + EXPECT_FALSE(bycorf::RegisteredMemorySlice{} + .RemoteRegion(RdmaAccess::kRemoteRead) + .ok()); +} + +TEST_F(RdmaMemory, SliceAloneRetainsRegistrationAndMovedViewIsEmpty) { + std::array backing; + bycorf::Worker worker; + auto domain = worker.OpenRdmaDevice("test0"); + ASSERT_TRUE(domain.ok()); + auto mr = worker.RegisterRdmaMemory( + *domain, backing, RdmaAccess::kLocalWrite | RdmaAccess::kRemoteAtomic); + ASSERT_TRUE(mr.ok()); + EXPECT_EQ(verbs.last_access, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_ATOMIC); + auto slice = mr->Slice(0, 64); + ASSERT_TRUE(slice.ok()); + mr = bycorf::RegisteredMemory{}; + EXPECT_EQ(verbs.mrs, 1); + auto moved = std::move(*slice); + EXPECT_EQ(slice->size(), 0); + EXPECT_FALSE(slice->Slice(0, 1).ok()); + EXPECT_TRUE(moved.RemoteRegion(RdmaAccess::kRemoteAtomic).ok()); + moved = {}; + EXPECT_EQ(verbs.mrs, 0); +} + +TEST_F(RdmaMemory, RemoteRangesCannotOverflow) { + bycorf::RemoteMemoryRegion region{100, 50, 7}; + auto sub = region.Slice(10, 40); + ASSERT_TRUE(sub.ok()); + EXPECT_EQ(sub->address, 110); + EXPECT_EQ(sub->length, 40); + EXPECT_EQ(sub->rkey, 7); + EXPECT_FALSE(region.Slice(51, 1).ok()); + EXPECT_FALSE(region.Slice(0, 0).ok()); + EXPECT_FALSE(region.Slice(1, std::numeric_limits::max()).ok()); + region.address = std::numeric_limits::max() - 1; + EXPECT_FALSE(region.Slice(0, 1).ok()); +} +} // namespace diff --git a/tests/rdma_registration_test.cpp b/tests/rdma_registration_test.cpp new file mode 100644 index 0000000..61bc61b --- /dev/null +++ b/tests/rdma_registration_test.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2026 EloqData Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include "bycorf/runtime/worker.h" + +namespace { +using bycorf::RdmaAccess; +using bycorf::RdmaLocalAccess; + +TEST(RdmaRegistration, SoftRoceRegistrationAndQpSharing) { + const char* name = std::getenv("BYCORF_RDMA_TEST_DEVICE"); + if (!name || !*name) GTEST_SKIP() << "Set BYCORF_RDMA_TEST_DEVICE to opt in"; + // Backing memory is declared first: it survives every registration reference. + std::array backing{}; + bycorf::Worker worker, other; + auto domain = worker.OpenRdmaDevice(name); + ASSERT_TRUE(domain.ok()) << domain.status(); + auto again = worker.OpenRdmaDevice(name); + ASSERT_TRUE(again.ok()); + EXPECT_EQ(domain->native_pd(), again->native_pd()); + auto foreign = other.OpenRdmaDevice(name); + ASSERT_TRUE(foreign.ok()) << foreign.status(); + EXPECT_NE(domain->native_pd(), foreign->native_pd()); + EXPECT_FALSE( + worker.RegisterRdmaMemory(*foreign, backing, RdmaAccess::kNone).ok()); + auto mr = worker.RegisterRdmaMemory(*domain, backing, + RdmaAccess::kLocalWrite | + RdmaAccess::kRemoteRead | + RdmaAccess::kRemoteWrite); + ASSERT_TRUE(mr.ok()) << mr.status(); + auto slice = mr->Slice(64, 128); + ASSERT_TRUE(slice.ok()); + EXPECT_FALSE(mr->Slice(4090, 7).ok()); + auto remote = slice->RemoteRegion(RdmaAccess::kRemoteWrite); + ASSERT_TRUE(remote.ok()); + EXPECT_EQ(remote->address, reinterpret_cast(backing.data() + 64)); + EXPECT_EQ(remote->length, 128); + auto* pd = domain->native_pd(); + std::unique_ptr cq( + ibv_create_cq(pd->context, 16, nullptr, nullptr, 0), ibv_destroy_cq); + ASSERT_NE(cq, nullptr); + ibv_qp_init_attr attr{}; + attr.send_cq = attr.recv_cq = cq.get(); + attr.qp_type = IBV_QPT_RC; + attr.cap.max_send_wr = attr.cap.max_recv_wr = 4; + attr.cap.max_send_sge = attr.cap.max_recv_sge = 1; + std::unique_ptr first( + ibv_create_qp(pd, &attr), ibv_destroy_qp); + std::unique_ptr second( + ibv_create_qp(pd, &attr), ibv_destroy_qp); + ASSERT_NE(first, nullptr); + ASSERT_NE(second, nullptr); + auto lease = slice->Acquire(*first, RdmaLocalAccess::kWrite); + ASSERT_TRUE(lease.ok()); + EXPECT_TRUE(slice->Acquire(*second, RdmaLocalAccess::kWrite).ok()); + auto read_only = + worker.RegisterRdmaMemory(*domain, backing, RdmaAccess::kNone); + ASSERT_TRUE(read_only.ok()); + auto readonly_slice = read_only->Slice(0, 64); + ASSERT_TRUE(readonly_slice.ok()); + EXPECT_TRUE(readonly_slice->Acquire(*first, RdmaLocalAccess::kRead).ok()); + EXPECT_TRUE(absl::IsPermissionDenied( + readonly_slice->Acquire(*first, RdmaLocalAccess::kWrite).status())); + EXPECT_TRUE(absl::IsPermissionDenied( + readonly_slice->RemoteRegion(RdmaAccess::kRemoteRead).status())); + EXPECT_FALSE( + worker.RegisterRdmaMemory(*domain, backing, RdmaAccess::kRemoteWrite) + .ok()); + EXPECT_FALSE( + worker.RegisterRdmaMemory(*domain, backing, RdmaAccess::kRemoteAtomic) + .ok()); + // Another real PD on the same actual context must fail the SGE check. + auto context = std::shared_ptr( + std::make_shared(*domain), pd->context); + auto other_pd = other.AcquireRdmaDomain(context); + ASSERT_TRUE(other_pd.ok()); + std::unique_ptr wrong( + ibv_create_qp(other_pd->native_pd(), &attr), ibv_destroy_qp); + ASSERT_NE(wrong, nullptr); + EXPECT_FALSE(slice->Acquire(*wrong, RdmaLocalAccess::kRead).ok()); + // Drop public registration handles: the operation lease still owns the MR. + mr = bycorf::RegisteredMemory{}; + slice = bycorf::RegisteredMemorySlice{}; + EXPECT_EQ(lease->sge().length, 128); + lease = bycorf::RdmaMemoryLease{}; + // Exercise deregistration/re-registration repeatedly on the actual provider. + for (int i = 0; i < 32; ++i) { + auto transient = + worker.RegisterRdmaMemory(*domain, backing, RdmaAccess::kLocalWrite); + ASSERT_TRUE(transient.ok()) << transient.status(); + } +} +} // namespace diff --git a/tests/worker_wait_test.cpp b/tests/worker_wait_test.cpp index 2e751f7..22e6633 100644 --- a/tests/worker_wait_test.cpp +++ b/tests/worker_wait_test.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include @@ -86,6 +87,44 @@ TEST_F(WorkerWait, NetworkFacadeDoesNotAdvanceSharedRing) { CheckNetworkRingIsolation(workers[0]); } +#if BYCORF_ENABLE_RDMA +TEST_F(WorkerWait, RdmaFacadeUsesSharedRingForReadinessAndCancellation) { + struct Completion : IoCompletion { + unsigned calls = 0; + int result = 0; + void Complete(Worker&, int value, unsigned) override { + ++calls; + result = value; + } + } ready, cancelled; + int fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + ASSERT_GE(fd, 0); + auto& kernel = WorkerWaitTestPeer::Kernel(workers[0]); + ASSERT_TRUE(workers[0].SubmitRdmaEventPoll(fd, POLLIN, &ready).ok()); + const std::uint64_t one = 1; + ASSERT_EQ(write(fd, &one, sizeof(one)), sizeof(one)); + // Only the existing kernel ring is driven. A separate RDMA ring would leave + // this tag pending until the fixture's lost-wakeup alarm fired. + while (!ready.calls) { + ASSERT_TRUE(kernel.Submit().ok()); + kernel.Poll(); + } + EXPECT_EQ(ready.calls, 1U); + EXPECT_EQ(ready.result, POLLIN); + std::uint64_t count; + ASSERT_EQ(read(fd, &count, sizeof(count)), sizeof(count)); + ASSERT_TRUE(workers[0].SubmitRdmaEventPoll(fd, POLLIN, &cancelled).ok()); + ASSERT_TRUE(workers[0].CancelRdmaEventPoll(&cancelled).ok()); + while (!cancelled.calls) { + ASSERT_TRUE(kernel.Submit().ok()); + kernel.Poll(); + } + EXPECT_EQ(cancelled.calls, 1U); + EXPECT_EQ(cancelled.result, -ECANCELED); + close(fd); +} +#endif + TEST_F(WorkerWait, MailboxPublicationAfterEmptyCheckDefeatsPark) { for (bool foreign : {false, true}) { bool waiting = false;