Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ option(BYCORF_ENABLE_WERROR "Treat warnings as errors" OFF)
option(BYCORF_KERNEL_BYPASS
"Include DPDK networking and SPDK storage (selected independently at startup)"
OFF)
option(BYCORF_ENABLE_RDMA "Include RDMA facade and memory registration" OFF)
option(BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE
"Enable detailed cross-core queue and wake latency tracing" OFF)

Expand Down Expand Up @@ -174,11 +175,22 @@ add_library(bycorf

target_compile_definitions(bycorf PUBLIC
# Public headers and consumers must agree on the backend-dependent layouts.
# Export one numeric switch for both capabilities, including zero when off.
# Export numeric capability switches, including zero when disabled.
BYCORF_KERNEL_BYPASS=$<BOOL:${BYCORF_KERNEL_BYPASS}>
BYCORF_ENABLE_RDMA=$<BOOL:${BYCORF_ENABLE_RDMA}>
BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE=$<BOOL:${BYCORF_ENABLE_CROSS_CORE_LATENCY_TRACE}>
)

# RDMA is an independent capability: do not discover its libraries when off.
if (BYCORF_ENABLE_RDMA)
find_package(PkgConfig REQUIRED)
pkg_check_modules(BYCORF_IBVERBS REQUIRED IMPORTED_TARGET libibverbs)
pkg_check_modules(BYCORF_RDMACM REQUIRED IMPORTED_TARGET librdmacm)
target_sources(bycorf PRIVATE src/io/rdma_memory.cpp)
target_link_libraries(bycorf PUBLIC
PkgConfig::BYCORF_IBVERBS PkgConfig::BYCORF_RDMACM)
endif()

if (BYCORF_KERNEL_BYPASS)
include(cmake/Dpdk.cmake)
set(BYCORF_SPDK_SOURCE_DIR
Expand Down Expand Up @@ -465,6 +477,21 @@ if (BYCORF_BUILD_TESTS)
target_link_libraries(bycorf_unit_tests PRIVATE bycorf::core GTest::gtest_main)
bycorf_discover_tests(bycorf_unit_tests 30 "runtime;unit")

if (BYCORF_ENABLE_RDMA)
add_executable(bycorf_rdma_memory_tests tests/rdma_memory_test.cpp)
target_link_libraries(bycorf_rdma_memory_tests PRIVATE bycorf::core GTest::gtest_main)
foreach(symbol IN ITEMS ibv_get_device_list ibv_free_device_list
ibv_get_device_name ibv_open_device ibv_close_device ibv_alloc_pd
ibv_dealloc_pd ibv_reg_mr ibv_reg_mr_iova2 ibv_dereg_mr)
target_link_options(bycorf_rdma_memory_tests PRIVATE "LINKER:--wrap=${symbol}")
endforeach()
bycorf_discover_tests(bycorf_rdma_memory_tests 30 "runtime;rdma;unit")

add_executable(bycorf_rdma_registration_tests tests/rdma_registration_test.cpp)
target_link_libraries(bycorf_rdma_registration_tests PRIVATE bycorf::core GTest::gtest_main)
bycorf_discover_tests(bycorf_rdma_registration_tests 30 "runtime;rdma;integration")
endif()

add_executable(bycorf_dpdk_memory_budget_tests
tests/dpdk_memory_budget_test.cpp src/io/dpdk_memory_budget.cpp)
target_link_libraries(bycorf_dpdk_memory_budget_tests PRIVATE
Expand Down
41 changes: 41 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ connections, and asynchronous I/O; applications provide services above

- [Name](#name)
- [Build and test](#build-and-test)
- [Optional RDMA registration](#optional-rdma-registration)
- [Run the examples](#run-the-examples)
- [Use Bycorf in a CMake project](#use-bycorf-in-a-cmake-project)
- [Development](#development)
Expand Down Expand Up @@ -178,6 +179,46 @@ Follow the [DPDK/SPDK build and test guide](docs/dpdk-prototype.md) for its
additional dependencies, virtual-device checks, physical-device configuration,
and experimental compatibility limits.

## Optional RDMA registration

`BYCORF_ENABLE_RDMA` enables the private worker facade and explicit memory
registration API. It defaults to off and is independent of
`BYCORF_KERNEL_BYPASS`; enabling it requires libibverbs and librdmacm development
packages, without DPDK/SPDK. On Ubuntu:

```bash
sudo apt-get install --no-install-recommends libibverbs-dev librdmacm-dev
cmake -S . -B build-rdma -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBYCORF_ENABLE_RDMA=ON
cmake --build build-rdma --parallel
ctest --test-dir build-rdma -R '^bycorf_RdmaMemory\.' --output-on-failure
```

The device-free `RdmaMemory` suite injects verbs failures and verifies lazy
initialization, cleanup order, range/permission checks, worker/context identity,
and retained registration lifetimes. The `RdmaRegistration` test skips unless
an explicit device is supplied. With an existing Soft-RoCE device (substitute
its name from `rdma link show`):

```bash
BYCORF_RDMA_TEST_DEVICE=rxe0 ctest --test-dir build-rdma \
-R '^bycorf_RdmaRegistration\.' --output-on-failure
```

This opens the selected device, registers and deregisters memory, creates real
QPs sharing a PD, and checks permissions, bounds, and mismatched PD rejection.
It requires access to the device's uverbs node and sufficient memlock. It does
not test a connected data-transfer or immediate-data protocol. No device is
opened by unused workers or test discovery.

Use `Worker::OpenRdmaDevice(name)` or `AcquireRdmaDomain(owning_context)` on the
owner worker, then `RegisterRdmaMemory(domain, backing_span, explicit_access)`.
Keep the returned domain through QP destruction. `RegisteredMemory::Slice`
retains registration; `slice.Acquire(qp, local_access)` validates context/PD and
returns an SGE lease to retain through terminal completion. The caller owns
backing memory and must stop remote access before releasing registrations and
freeing it. See [RDMA ownership](docs/architecture/rdma.md) for remote descriptors,
permissions, CM context ownership, and the independent storage-buffer contract.

## Run the examples

The default build enables `BYCORF_BUILD_EXAMPLES`. If it was disabled, set
Expand Down
4 changes: 3 additions & 1 deletion docs/architecture/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ and asynchronous I/O. Abseil supplies status types and spdlog supplies logging.
| Module | Responsibility | Document |
|---|---|---|
| Runtime | Worker lifecycle, coroutine scheduling, sleep coordination, cross-worker and foreign ingress | [Runtime](runtime.md) |
| I/O | Worker-owned shared io_uring, network/storage adapters, optional SPDK storage, shared DPDK environment | [I/O backends](io.md) |
| I/O | Worker-owned shared io_uring, network/storage adapters, optional RDMA registration and SPDK storage, shared DPDK environment | [I/O backends](io.md), [RDMA ownership](rdma.md) |
| Networking | Listeners, streams, services, TLS, optional DPDK/FreeBSD IPv4 TCP | [Networking](networking.md) |
| RPC | Framed request/reply protocol above streams | [RPC](rpc.md) |

Expand All @@ -38,6 +38,8 @@ worker before that coroutine uses its stream again.
Startup selection independently activates network and storage backends and
remains fixed for the process lifetime. Both default to io_uring even in a
bypass-capable build.
`BYCORF_ENABLE_RDMA` independently enables the RDMA facade and registration
layer; resources are acquired only on explicit use.
The [prototype runbook](../dpdk-prototype.md) describes the DPDK configuration.

Architecture documents describe the current core model and stable tradeoffs.
Expand Down
12 changes: 8 additions & 4 deletions docs/architecture/io.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ limitations under the License.

`Worker` owns one `IoUringBackend` for kernel sockets, file I/O, completion
delivery, timers, and MSG_RING/eventfd wakeups. Its `NetBackend<Worker>` and
`StorageBackend<Worker>` facades share that ring. Network receive buffers and
registered storage buffers have separate lifecycles. Linux TCP uses
`StorageBackend<Worker>` facades share that ring. The independently optional
`RdmaBackend<Worker>` also uses this ring for channel readiness. Network receive
buffers, registered storage buffers and RDMA registrations have separate
lifecycles; see [RDMA ownership](rdma.md). Linux TCP uses
provided-buffer multishot receives with a per-connection one-shot fallback.

`BYCORF_KERNEL_BYPASS` is the single build option for DPDK networking and SPDK
Expand All @@ -42,8 +44,10 @@ Network adapters supply notification preparation, idle checks,
deadlines, and cleanup to the [worker sleep protocol](runtime.md#sleep-and-wakeups).
The worker enters the ring's blocking wait; adapters never block independently.

The ring outlives both adapters. The worker initializes it first, stops storage
and networking in that order, then shuts it down. Storage dispatch remains
The ring outlives all facades. RDMA drops its weak resource cache on shutdown;
registration and operation handles retain their PD/context independently. The
worker initializes the ring first, stops storage and networking in that order,
then shuts the ring down. Storage dispatch remains
valid through shutdown completion draining; SPDK callbacks use the owning
worker's TLS to reach its storage adapter. Kernel registrations remain live
until ring shutdown quiesces kernel access.
Expand Down
70 changes: 70 additions & 0 deletions docs/architecture/rdma.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
<!--
Copyright (C) 2026 EloqData Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

# RDMA registration and ownership

`BYCORF_ENABLE_RDMA` adds an independent I/O capability backed by libibverbs and
librdmacm. It defaults to off and does not select or require DPDK/SPDK. The
private `RdmaBackend<Worker>` facade reaches the worker's existing io_uring
through `KernelBackend()` for channel readiness and cancellation. It creates no
ring, polling thread, thread pool, or process-wide blocking lock. The worker
continues to own shared-ring submission and completion dispatch.

Construction does not enumerate or open devices. Explicit `OpenRdmaDevice`
selects a named verbs device; `AcquireRdmaDomain` accepts an owning reference to
the actual context selected by the caller, including a CM-selected context.
An aliasing shared pointer can keep the CM context owner alive. A borrowed raw
context with a no-op deleter is insufficient. Resource acquisition and cache
access occur on the owning worker. Each worker caches live PDs by actual
context identity; two contexts for one device and two workers using one context
have distinct PDs. The cache holds weak references, so resource ownership is
independent of worker shutdown. Partial initialization unwinds in reverse order.

`RdmaProtectionDomain` retains its PD and context. QP owners create QPs in that
PD and retain the domain until QP destruction. A registered MR can be used by
multiple QPs in the same PD; it belongs to neither a particular QP nor an IMM
(immediate-data) operation. Operation leases check both the QP's context and PD.
Connection management, QP posting and completion protocols are outside this
registration layer.

`RegisteredMemory` registers caller-owned backing memory with explicit access
permissions. Copies, registered slices, and operation leases retain the same
MR. The last reference deregisters it before releasing its PD and context.
An operation keeps its lease until terminal completion, including error and
cancellation; copying the SGE alone does not retain registration. The application
keeps backing memory alive through all local completions and remote quiescence,
then releases every registration reference before freeing the memory. Dropping
a registration handle or shutting down a worker does not stop remote access.
Teardown failures terminate rather than release dependencies still potentially
used by hardware.

Slices validate nonempty ranges without arithmetic overflow. Local reads need
no access flag; local DMA writes require local-write permission. Remote write
and atomic registration also require local-write permission. Unsupported access
bits are rejected. `RemoteMemoryRegion{address, length, rkey}` is a non-owning
peer descriptor with checked subranges, not a prescribed wire encoding. Export
requires explicit granted remote permissions. Its length is an application
contract: a slice does not narrow the hardware rkey's authority over the MR,
and exporting a subset of permissions does not reduce the MR's granted rights.

RDMA registration does not reuse storage `RegisterBuffers`, allocate backing
memory, grow arenas, or supply a business buffer pool. Memory used by both
storage and RDMA must independently satisfy both registrations' alignment,
access and lifetime requirements.

Sources: `include/bycorf/io/rdma_backend.h`,
`include/bycorf/io/rdma_memory.h`, `src/io/rdma_memory.cpp`,
`include/bycorf/runtime/worker.h`, `src/runtime/worker.cpp`.
75 changes: 75 additions & 0 deletions include/bycorf/io/rdma_backend.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
* Copyright (C) 2026 EloqData Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef BYCORF_IO_RDMA_BACKEND_H_
#define BYCORF_IO_RDMA_BACKEND_H_

#if BYCORF_ENABLE_RDMA
#include "bycorf/io/io_uring_backend.h"
#include "bycorf/io/rdma_memory.h"

namespace bycorf {
// Private CRTP facade: the owner supplies its existing KernelBackend(). Neither
// construction nor destruction downcasts or accesses devices/the shared ring.
// All calls run on the owning worker; external handles can outlive the facade.
template <typename Owner>
class RdmaBackend {
public:
// Explicit device selection; no default-device fallback. Opens lazily and
// reuses a live worker-local PD for this context/device.
absl::StatusOr<RdmaProtectionDomain> OpenRdmaDevice(std::string_view name) {
return resources_.OpenDevice(name);
}
// For a CM-selected context, pass an owning shared_ptr (an aliasing pointer
// may retain its CM owner). It must keep the exact context alive; wrapping a
// borrowed pointer with a no-op deleter does not satisfy this contract.
absl::StatusOr<RdmaProtectionDomain> AcquireRdmaDomain(
const std::shared_ptr<ibv_context>& context) {
return resources_.Domain(context);
}
// Rejects domains from another worker, invalid ranges and invalid access.
// Caller owns memory; this API neither allocates nor grows backing buffers.
absl::StatusOr<RegisteredMemory> RegisterRdmaMemory(
const RdmaProtectionDomain& domain, std::span<std::byte> memory,
RdmaAccess access) {
return resources_.Register(domain, memory, access);
}
// Channel readiness uses the owner's single ring. The caller retains fd/tag
// through terminal completion, including cancellation, as for kernel I/O.
absl::Status SubmitRdmaEventPoll(int fd, unsigned events, IoCompletion* tag) {
return Kernel().SubmitPoll(fd, events, tag);
}
absl::Status CancelRdmaEventPoll(IoCompletion* tag) {
return Kernel().SubmitCancel(tag);
}
// Drops only cache entries. Outstanding handles continue to own resources;
// callers must quiesce QPs/remote access before releasing those handles.
void Shutdown() noexcept { resources_.Shutdown(); }

private:
friend Owner;
RdmaBackend() = default;
~RdmaBackend() = default;
RdmaBackend(const RdmaBackend&) = delete;
RdmaBackend& operator=(const RdmaBackend&) = delete;
IoUringBackend& Kernel() noexcept {
return static_cast<Owner&>(*this).KernelBackend();
}
detail::RdmaResources resources_;
};
} // namespace bycorf
#endif // BYCORF_ENABLE_RDMA
#endif // BYCORF_IO_RDMA_BACKEND_H_
Loading