Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,11 @@ if (BYCORF_BUILD_TESTS)
target_include_directories(bycorf_dpdk_memory_budget_tests PRIVATE src/io)
bycorf_discover_tests(bycorf_dpdk_memory_budget_tests 30 "runtime;net;unit")

add_executable(bycorf_dpdk_socket_table_tests tests/dpdk_socket_table_test.cpp)
target_link_libraries(bycorf_dpdk_socket_table_tests PRIVATE GTest::gtest_main)
target_include_directories(bycorf_dpdk_socket_table_tests PRIVATE src/io)
bycorf_discover_tests(bycorf_dpdk_socket_table_tests 30 "runtime;net;unit")

add_executable(bycorf_task_tests tests/task_transfer_test.cpp)
target_link_libraries(bycorf_task_tests PRIVATE GTest::gtest)
target_include_directories(bycorf_task_tests PRIVATE include)
Expand All @@ -491,6 +496,14 @@ if (BYCORF_BUILD_TESTS)
bycorf_discover_tests(bycorf_runtime_tests 90 "runtime;net;io_uring")

if (BYCORF_KERNEL_BYPASS)
add_executable(bycorf_dpdk_handle_reuse_tests tests/dpdk_handle_reuse_test.cpp)
target_link_libraries(bycorf_dpdk_handle_reuse_tests PRIVATE
bycorf::core GTest::gtest_main)
target_include_directories(bycorf_dpdk_handle_reuse_tests PRIVATE src/io)
target_link_options(bycorf_dpdk_handle_reuse_tests PRIVATE
"LINKER:--wrap=bycorf_bsd_send" "LINKER:--wrap=bycorf_bsd_connect")
bycorf_discover_tests(bycorf_dpdk_handle_reuse_tests 30 "runtime;net;dpdk;stub")

add_executable(bycorf_dpdk_memory_startup_tests
tests/dpdk_memory_startup_test.cpp)
target_link_libraries(bycorf_dpdk_memory_startup_tests PRIVATE
Expand Down
10 changes: 8 additions & 2 deletions docs/architecture/networking.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,13 @@ bounds checks remain active; `tests/freebsd_tcp_sequence_check.cpp` covers this
invariant through the real stack and an in-process Ethernet peer.

BSD sockets use private worker-encoded handles rather than Linux file
descriptors. Bycorf's close and peer-address helpers dispatch these handles to
the stack. Applications that directly pass stream handles to Linux syscalls or
descriptors. Handle slots are reusable within their owning worker. Closing a
socket immediately invalidates its transport; pending operations and queued
completions retain the old slot through callback delivery before it can be
assigned to another socket. Thus handle capacity bounds simultaneously live
or draining sockets, not cumulative connection churn. Bycorf's close and
peer-address helpers dispatch these handles to the stack. Applications that
directly pass stream handles to Linux syscalls or
transfer a live socket to another worker need explicit adaptation. Accepted
and outgoing IPv4 TCP use BSD; Unix sockets retain the kernel path. IPv6 DPDK
listeners and clients are unsupported.
Expand Down Expand Up @@ -179,4 +184,5 @@ both modes; this is separate from Bycorf's worker sleep protocol.

Sources: `include/bycorf/net/connection.h`, `src/net/tcp_service.cpp`, `src/net/tcp_listener.cpp`,
`src/net/tcp_stream.cpp`, `src/net/socket_ops.cpp`, `src/io/dpdk_backend.cpp`,
`src/io/dpdk_socket_table.h`,
`src/io/dpdk_memory_budget.cpp`, `src/io/freebsd/abi.h`, `src/io/freebsd/`.
2 changes: 2 additions & 0 deletions include/bycorf/io/dpdk_backend.h
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ class DpdkBackend {
// Socket helpers require the owning worker thread. Return host errno values.
static int Listen(const sockaddr* address, socklen_t length, int backlog);
static int OpenClient(int family);
// Closes transport immediately. Pending operations retain the handle slot
// until their completions retire, after which another socket may reuse it.
static int Close(int handle) noexcept;
static int PeerName(int handle, sockaddr*, socklen_t*) noexcept;

Expand Down
124 changes: 76 additions & 48 deletions src/io/dpdk_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
#include "bycorf/net/socket_ops.h"
#include "bycorf/runtime/worker.h"
#include "dpdk_memory_budget.h"
#include "dpdk_socket_table.h"
#include "freebsd/abi.h"
#include "freebsd_errno.h"

Expand All @@ -65,7 +66,7 @@ static_assert(kMaxWorkers > 0 && kMaxWorkers < RTE_MAX_LCORE);
constexpr int kFirstHandle = 0x40000000;
// Bit 30 distinguishes BSD handles from kernel fds; the remaining positive
// bits partition the namespace by owner. Fixed 24-bit slots overflowed signed
// handles at worker 64. Leave the last slot unused so increment never wraps.
// handles at worker 64. Keep allocation inside each worker's slot range.
constexpr unsigned kHandleBits = 30 - std::bit_width(kMaxWorkers - 1);
constexpr unsigned kHandleMask = (1U << kHandleBits) - 1;
constexpr std::size_t kMaxFrame = 1518;
Expand Down Expand Up @@ -224,11 +225,11 @@ class DpdkBackend::Impl {
unsigned id = 0;
bool initialized = false, polling = false;
std::size_t buffer_size = 4096;
int next_handle = kFirstHandle;
unsigned next_client_port = 0;
std::unordered_map<int, BsdSocket*> sockets;
detail::DpdkSocketTable sockets;
using SocketPin = detail::DpdkSocketTable::Pin;
struct Request {
int fd;
SocketPin socket;
IoCompletion* tag;
std::span<const std::byte> bytes;
const msghdr* message = nullptr;
Expand All @@ -237,9 +238,14 @@ class DpdkBackend::Impl {
IoCompletion* tag;
int result;
unsigned flags;
SocketPin socket;
};
struct ConnectionRequest {
Connection* connection;
SocketPin socket;
};
std::vector<Request> accepts, sends, connects;
std::vector<Connection*> reads, disconnects;
std::vector<ConnectionRequest> reads, disconnects;
std::vector<Completion> completions;
std::uint64_t rx = 0, tx = 0, forwarded_rx = 0, forwarded_tx = 0, drops = 0;
std::uint64_t waits = 0, arms = 0, notifications = 0;
Expand Down Expand Up @@ -275,23 +281,21 @@ class DpdkBackend::Impl {
}
} notification;

BsdSocket* Find(int handle) {
const auto found = sockets.find(handle);
return found == sockets.end() ? nullptr : found->second;
}
BsdSocket* Find(int handle) { return sockets.Find(handle); }
int Own(BsdSocket* so) {
if ((next_handle & kHandleMask) == kHandleMask) {
const int handle = sockets.Insert(so);
if (handle < 0) {
bycorf_bsd_close(so);
errno = EMFILE;
return -1;
}
const int handle = next_handle++;
sockets.emplace(handle, so);
return handle;
}
void QueueCompletion(IoCompletion* tag, int result,
unsigned flags = kCompletionNone) {
completions.push_back({tag, result, flags});
unsigned flags = kCompletionNone,
SocketPin socket = {}) {
// A callback may close a socket and create another one. Retain its old
// number through callback delivery, not merely removal from a pending list.
completions.push_back({tag, result, flags, std::move(socket)});
}
void Wake(unsigned target) {
Lane& lane = fabric.lanes[target];
Expand Down Expand Up @@ -466,21 +470,22 @@ class DpdkBackend::Impl {
bool SocketCompletions() {
bool work = false;
for (std::size_t i = 0; i < connects.size();) {
const auto request = connects[i];
BsdSocket* so = Find(request.fd);
auto request = connects[i];
BsdSocket* so = request.socket.get();
const int error =
so ? BsdError(bycorf_bsd_connect_status(so)) : ECANCELED;
if (error == EAGAIN) {
++i;
continue;
}
connects.erase(connects.begin() + i);
QueueCompletion(request.tag, -error);
QueueCompletion(request.tag, -error, kCompletionNone,
std::move(request.socket));
work = true;
}
for (std::size_t i = 0; i < accepts.size();) {
const auto request = accepts[i];
BsdSocket* listener = Find(request.fd);
auto request = accepts[i];
BsdSocket* listener = request.socket.get();
BsdSocket* accepted = nullptr;
const int error =
listener ? BsdError(bycorf_bsd_accept(listener, &accepted)) : EBADF;
Expand All @@ -491,17 +496,18 @@ class DpdkBackend::Impl {
if (!error) {
const int handle = Own(accepted);
QueueCompletion(request.tag, handle < 0 ? -errno : handle,
kCompletionMore);
kCompletionMore, std::move(request.socket));
++i;
} else {
accepts.erase(accepts.begin() + i);
QueueCompletion(request.tag, -error);
QueueCompletion(request.tag, -error, kCompletionNone,
std::move(request.socket));
}
work = true;
}
for (std::size_t i = 0; i < sends.size();) {
const auto request = sends[i];
BsdSocket* so = Find(request.fd);
auto request = sends[i];
BsdSocket* so = request.socket.get();
std::size_t sent = 0;
int error = so ? 0 : EBADF;
if (so && request.message) {
Expand All @@ -525,12 +531,14 @@ class DpdkBackend::Impl {
continue;
}
sends.erase(sends.begin() + i);
QueueCompletion(request.tag, sent ? static_cast<int>(sent) : -error);
QueueCompletion(request.tag, sent ? static_cast<int>(sent) : -error,
kCompletionNone, std::move(request.socket));
work = true;
}
for (std::size_t i = 0; i < reads.size();) {
Connection* c = reads[i];
BsdSocket* so = Find(c->file_.fd_);
const auto request = reads[i];
Connection* c = request.connection;
BsdSocket* so = request.socket.get();
std::size_t got = 0;
const int error =
so && !c->closing_
Expand All @@ -554,8 +562,9 @@ class DpdkBackend::Impl {
work = true;
}
for (std::size_t i = 0; i < disconnects.size();) {
Connection* c = disconnects[i];
BsdSocket* so = Find(c->file_.fd_);
const auto request = disconnects[i];
Connection* c = request.connection;
BsdSocket* so = request.socket.get();
const bool cancel =
c->peer_disconnect_poll_cancel_requested_ || c->closing_ || !so;
if (!cancel && !bycorf_bsd_disconnected(so)) {
Expand Down Expand Up @@ -870,7 +879,8 @@ absl::Status DpdkBackend::Init(const IoBackendOptions& options,
impl_->buffer_size = options.recv_buffer_size_;
// Embed the owner in the handle so an accidental transfer cannot alias an
// unrelated socket that happens to occupy the same slot on another worker.
impl_->next_handle = kFirstHandle | (impl_->id << kHandleBits);
impl_->sockets.Configure(kFirstHandle | (impl_->id << kHandleBits),
kHandleMask);
current_ = impl_.get();
if (rte_thread_register()) {
auto status = DeviceError("register worker with EAL", rte_errno);
Expand Down Expand Up @@ -921,8 +931,7 @@ void DpdkBackend::Shutdown() {
if (current_ != impl_.get())
std::abort(); // Runtime tears down on the owning thread.
fabric.lanes[impl_->id].parked.store(false, std::memory_order_release);
for (auto [handle, so] : impl_->sockets) bycorf_bsd_close(so);
impl_->sockets.clear();
impl_->sockets.CloseAll([](BsdSocket* so) { bycorf_bsd_close(so); });
impl_->SocketCompletions();
impl_->FlushTx();
std::fprintf(
Expand Down Expand Up @@ -1036,14 +1045,18 @@ absl::Status DpdkBackend::SubmitSend(const RegisteredFile& file,
if (!detail::IsDpdkSocket(file.fd_))
return kernel_.SubmitSend(file, bytes, tag);
if (!tag) return absl::InvalidArgumentError("send completion is null");
impl_->sends.push_back({file.fd_, tag, bytes});
auto socket = impl_->sockets.Acquire(file.fd_);
if (!socket)
return absl::FailedPreconditionError("send on closed DPDK socket");
impl_->sends.push_back({std::move(socket), tag, bytes});
return absl::OkStatus();
}
absl::Status DpdkBackend::SubmitConnect(int fd, const sockaddr* address,
socklen_t length, IoCompletion* tag) {
if (!detail::IsDpdkSocket(fd))
return kernel_.SubmitConnect(fd, address, length, tag);
auto* so = impl_->Find(fd);
auto socket = impl_->sockets.Acquire(fd);
auto* so = socket.get();
if (!so || !tag || !address || address->sa_family != AF_INET ||
length < sizeof(sockaddr_in)) {
return absl::InvalidArgumentError("invalid DPDK connect request");
Expand All @@ -1052,9 +1065,9 @@ absl::Status DpdkBackend::SubmitConnect(int fd, const sockaddr* address,
const int error = BsdError(
bycorf_bsd_connect(so, ipv4->sin_addr.s_addr, ntohs(ipv4->sin_port)));
if (error && error != EINPROGRESS) {
impl_->QueueCompletion(tag, -error);
impl_->QueueCompletion(tag, -error, kCompletionNone, std::move(socket));
} else {
impl_->connects.push_back({fd, tag, {}});
impl_->connects.push_back({std::move(socket), tag, {}});
}
return absl::OkStatus();
}
Expand All @@ -1065,22 +1078,28 @@ absl::Status DpdkBackend::SubmitSendMsg(const RegisteredFile& file,
return kernel_.SubmitSendMsg(file, message, tag);
if (!tag || !message || message->msg_control || message->msg_name)
return absl::InvalidArgumentError("unsupported DPDK sendmsg request");
impl_->sends.push_back({file.fd_, tag, {}, message});
auto socket = impl_->sockets.Acquire(file.fd_);
if (!socket)
return absl::FailedPreconditionError("sendmsg on closed DPDK socket");
impl_->sends.push_back({std::move(socket), tag, {}, message});
return absl::OkStatus();
}
absl::Status DpdkBackend::SubmitAcceptMultishot(int fd, IoCompletion* tag) {
if (!detail::IsDpdkSocket(fd)) return kernel_.SubmitAcceptMultishot(fd, tag);
if (!tag || !impl_->Find(fd))
auto socket = impl_->sockets.Acquire(fd);
if (!tag || !socket)
return absl::InvalidArgumentError("invalid DPDK accept request");
impl_->accepts.push_back({fd, tag, {}});
impl_->accepts.push_back({std::move(socket), tag, {}});
return absl::OkStatus();
}
absl::Status DpdkBackend::SubmitCancel(IoCompletion* target) {
for (auto* requests : {&impl_->accepts, &impl_->sends, &impl_->connects}) {
for (auto at = requests->begin(); at != requests->end(); ++at) {
if (at->tag != target) continue;
auto socket = std::move(at->socket);
requests->erase(at);
impl_->QueueCompletion(target, -ECANCELED);
impl_->QueueCompletion(target, -ECANCELED, kCompletionNone,
std::move(socket));
return absl::OkStatus();
}
}
Expand All @@ -1092,34 +1111,44 @@ absl::Status DpdkBackend::StartRecvMultishot(Connection* c) {
if (c->recv_armed_ || c->closed_ || c->closing_ || c->recv_paused_ ||
!c->received_buffers_.empty())
return absl::OkStatus();
auto socket = impl_->sockets.Acquire(c->file_.fd_);
if (!socket)
return absl::FailedPreconditionError("recv on closed DPDK socket");
c->recv_mode_ = RecvMode::kOneShot;
if (c->read_buffer_.empty()) c->read_buffer_.resize(impl_->buffer_size);
c->recv_armed_ = true;
++c->inflight_ops_;
impl_->reads.push_back(c);
impl_->reads.push_back({c, std::move(socket)});
return absl::OkStatus();
}
absl::Status DpdkBackend::SubmitCancelRecv(Connection* c, IoCompletion* tag) {
const auto at = std::find(impl_->reads.begin(), impl_->reads.end(), c);
const auto at =
std::find_if(impl_->reads.begin(), impl_->reads.end(),
[c](const auto& r) { return r.connection == c; });
if (at == impl_->reads.end()) return kernel_.SubmitCancelRecv(c, tag);
auto socket = std::move(at->socket);
impl_->reads.erase(at);
impl_->FinishRead(c);
if (tag != nullptr) impl_->QueueCompletion(tag, 0);
if (tag != nullptr)
impl_->QueueCompletion(tag, 0, kCompletionNone, std::move(socket));
return absl::OkStatus();
}
absl::Status DpdkBackend::StartPeerDisconnectPoll(Connection* c) {
if (!c || !detail::IsDpdkSocket(c->file_.fd_))
return kernel_.StartPeerDisconnectPoll(c);
if (c->peer_disconnect_poll_armed_ || c->closed_ || c->closing_)
return absl::OkStatus();
auto socket = impl_->sockets.Acquire(c->file_.fd_);
if (!socket)
return absl::FailedPreconditionError("poll on closed DPDK socket");
c->peer_disconnect_poll_armed_ = true;
++c->inflight_ops_;
impl_->disconnects.push_back(c);
impl_->disconnects.push_back({c, std::move(socket)});
return absl::OkStatus();
}
absl::Status DpdkBackend::CancelPeerDisconnectPoll(Connection* c) {
if (std::find(impl_->disconnects.begin(), impl_->disconnects.end(), c) ==
impl_->disconnects.end())
if (std::none_of(impl_->disconnects.begin(), impl_->disconnects.end(),
[c](const auto& r) { return r.connection == c; }))
return kernel_.CancelPeerDisconnectPoll(c);
c->peer_disconnect_poll_cancel_requested_ = true;
return absl::OkStatus();
Expand Down Expand Up @@ -1192,12 +1221,11 @@ int DpdkBackend::Close(int handle) noexcept {
errno = EBADF;
return -1;
}
BsdSocket* so = current_->Find(handle);
BsdSocket* so = current_->sockets.Close(handle);
if (!so) {
errno = EBADF;
return -1;
}
current_->sockets.erase(handle);
const int error = BsdError(bycorf_bsd_close(so));
if (error) {
errno = error;
Expand Down
Loading