Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions src/io/spdk_storage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <spdk/nvme.h>

#include <algorithm>
#include <bit>
#include <cerrno>
#include <charconv>
#include <cstdio>
Expand Down Expand Up @@ -77,6 +78,9 @@ struct SpdkDevice {
SpdkController* controller_ = nullptr;
spdk_nvme_ns* ns_ = nullptr;
std::uint32_t sector_size_ = 0;
// Discovery accepts only nonzero powers of two. Cache their exponent so
// request validation and LBA conversion need no variable integer division.
unsigned sector_shift_ = 0;
std::uint64_t size_bytes_ = 0;
};

Expand Down Expand Up @@ -220,6 +224,7 @@ absl::StatusOr<SpdkDevice*> GetDevice(std::string_view path) {
device->controller_ = controller;
device->ns_ = ns;
device->sector_size_ = sector_size;
device->sector_shift_ = std::countr_zero(sector_size);
device->size_bytes_ = spdk_nvme_ns_get_size(ns);
SpdkDevice* raw = device.get();
g_spdk_devices.emplace(parsed->canonical_, std::move(device));
Expand All @@ -242,14 +247,14 @@ absl::Status SubmitSync(SpdkDevice& device, void* buffer, std::size_t bytes,
if (bytes == 0) {
return absl::OkStatus();
}
if (offset % device.sector_size_ != 0 || bytes % device.sector_size_ != 0 ||
if (((offset | bytes) & (device.sector_size_ - 1)) != 0 ||
offset > device.size_bytes_ || bytes > device.size_bytes_ - offset) {
return absl::Status(absl::StatusCode::kInvalidArgument,
"unaligned or out-of-range SPDK I/O");
}
const std::uint64_t lba = offset / device.sector_size_;
const std::uint64_t lba = offset >> device.sector_shift_;
const std::uint32_t lba_count =
static_cast<std::uint32_t>(bytes / device.sector_size_);
static_cast<std::uint32_t>(bytes >> device.sector_shift_);
SpdkController& controller = *device.controller_;
if (controller.sync_qpair_ == nullptr) {
return absl::Status(absl::StatusCode::kFailedPrecondition,
Expand Down Expand Up @@ -573,20 +578,22 @@ absl::Status SpdkBackend::SubmitIo(FixedFile file, void* buffer,
}
auto* device = static_cast<SpdkDevice*>(opened->device_);
const std::uint32_t sector = device->sector_size_;
const unsigned sector_shift = device->sector_shift_;
if (bytes > static_cast<std::size_t>(std::numeric_limits<int>::max()) ||
offset % sector != 0 || bytes % sector != 0 ||
bytes / sector > std::numeric_limits<std::uint32_t>::max() ||
((offset | bytes) & (sector - 1)) != 0 ||
(bytes >> sector_shift) > std::numeric_limits<std::uint32_t>::max() ||
offset > device->size_bytes_ || bytes > device->size_bytes_ - offset) {
return absl::Status(absl::StatusCode::kInvalidArgument,
"invalid SPDK I/O range or alignment");
}
const PendingIo request{.tag_ = tag,
.buffer_ = buffer,
.lba_ = offset / sector,
.count_ = static_cast<std::uint32_t>(bytes / sector),
.file_ = file,
.operation_ = write ? PendingIo::Operation::kWrite
: PendingIo::Operation::kRead};
const PendingIo request{
.tag_ = tag,
.buffer_ = buffer,
.lba_ = offset >> sector_shift,
.count_ = static_cast<std::uint32_t>(bytes >> sector_shift),
.file_ = file,
.operation_ =
write ? PendingIo::Operation::kWrite : PendingIo::Operation::kRead};
tag->success_result_ = static_cast<int>(bytes);
// Reuse the lookup needed for validation on the ordinary submission path;
// only a deferred retry needs to resolve the saved file index again.
Expand Down
145 changes: 143 additions & 2 deletions tests/spdk_backpressure_stub_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ namespace {
enum class Operation { kRead, kWrite, kFlush };
struct Namespace {
unsigned controller;
std::uint32_t sector_size = 512;
std::uint64_t size_bytes = 1ULL << 34;
};
struct Controller {
std::array<Namespace, 2> namespaces;
Expand Down Expand Up @@ -139,8 +141,12 @@ spdk_nvme_ns* __wrap_spdk_nvme_ctrlr_get_ns(spdk_nvme_ctrlr* ctrlr,
&reinterpret_cast<Controller*>(ctrlr)->namespaces[id - 1]);
}
bool __wrap_spdk_nvme_ns_is_active(spdk_nvme_ns*) { return true; }
uint32_t __wrap_spdk_nvme_ns_get_sector_size(spdk_nvme_ns*) { return 512; }
uint64_t __wrap_spdk_nvme_ns_get_size(spdk_nvme_ns*) { return 1ULL << 34; }
uint32_t __wrap_spdk_nvme_ns_get_sector_size(spdk_nvme_ns* ns) {
return reinterpret_cast<Namespace*>(ns)->sector_size;
}
uint64_t __wrap_spdk_nvme_ns_get_size(spdk_nvme_ns* ns) {
return reinterpret_cast<Namespace*>(ns)->size_bytes;
}
const spdk_nvme_ctrlr_data* __wrap_spdk_nvme_ctrlr_get_data(spdk_nvme_ctrlr*) {
static const auto data = [] {
spdk_nvme_ctrlr_data value{};
Expand Down Expand Up @@ -341,6 +347,96 @@ void CheckFixedBufferDescriptorLifetime(bycorf::Worker& worker,
CHECK(saved_read.lba == 4 && saved_read.count == 1);
}

void CheckSectorArithmetic(bycorf::Worker& worker,
std::span<std::byte> buffer) {
const auto sector = controllers[0].namespaces[0].sector_size;
const auto capacity = controllers[0].namespaces[0].size_bytes;
auto& queue = *qpairs[0];
Completion tail, write, read;
CHECK(worker.SubmitRead({0}, buffer, capacity - buffer.size(), &tail).ok());
CHECK(worker.SubmitWrite({1}, buffer, sector, &write).ok());
CHECK(worker.SubmitRead({0}, buffer, 2ULL * sector, &read).ok());
CHECK(queue.accepted.size() == 1);
CHECK(tail.calls + write.calls + read.calls == 0);
queue.completion_tokens = 3;
Drain(worker, [&] { return read.calls != 0; });
CHECK(tail.result == static_cast<int>(buffer.size()));
CHECK(write.result == static_cast<int>(buffer.size()));
CHECK(read.result == static_cast<int>(buffer.size()));
CHECK(queue.accepted.size() == 3);
CHECK(queue.accepted[0].lba == (capacity - buffer.size()) / sector);
CHECK(queue.accepted[1].operation == Operation::kWrite);
CHECK(queue.accepted[1].lba == 1 && queue.accepted[2].lba == 2);
for (const auto& command : queue.accepted) {
CHECK(command.buffer == buffer.data());
CHECK(command.count == buffer.size() / sector);
}

Completion rejected;
const auto attempts = queue.attempts;
if (sector > 1) {
CHECK(
absl::IsInvalidArgument(worker.SubmitRead({0}, buffer, 1, &rejected)));
CHECK(absl::IsInvalidArgument(worker.SubmitWrite(
{0}, buffer.first(buffer.size() - 1), 0, &rejected)));
}
CHECK(absl::IsInvalidArgument(
worker.SubmitRead({0}, buffer, capacity, &rejected)));
// Aligned offset whose addition with the request size would wrap. Bounds
// checks must continue to reject it before constructing a device command.
const auto overflowing_offset =
std::numeric_limits<std::uint64_t>::max() - (sector - 1);
CHECK(absl::IsInvalidArgument(
worker.SubmitWrite({0}, buffer, overflowing_offset, &rejected)));
CHECK(absl::IsInvalidArgument(worker.SubmitReadFixed(
{0}, {.data_ = buffer.data(), .size_ = 0}, 0, &rejected)));
CHECK(absl::IsInvalidArgument(worker.SubmitReadFixed(
{0},
{.data_ = buffer.data(),
.size_ = static_cast<std::size_t>(std::numeric_limits<int>::max()) + 1},
0, &rejected)));
CHECK(rejected.calls == 0 && queue.attempts == attempts);
}

void CheckSyncSectorArithmetic() {
CHECK(bycorf::ConfigureIoBackends({.spdk_storage = true}).ok());
CHECK(bycorf::ProbeSpdkStorage(paths[0]).ok());
CHECK(qpairs.size() == 1);
auto& queue = *qpairs[0];
const auto sector = controllers[0].namespaces[0].sector_size;
const auto capacity = controllers[0].namespaces[0].size_bytes;
// Link-time stubs do not DMA; direct metadata I/O needs no DMA allocator.
std::array<std::byte, 4096> buffer{};
queue.completion_tokens = 2;
CHECK(bycorf::spdk::sync::ReadDma(paths[0], buffer, capacity - buffer.size())
.ok());
CHECK(bycorf::spdk::sync::WriteDma(paths[0], buffer, sector, false).ok());
CHECK(queue.accepted.size() == 2);
CHECK(queue.accepted[0].lba == (capacity - buffer.size()) / sector);
CHECK(queue.accepted[1].operation == Operation::kWrite);
CHECK(queue.accepted[1].lba == 1);
for (const auto& command : queue.accepted) {
CHECK(command.buffer == buffer.data());
CHECK(command.count == buffer.size() / sector);
}
const auto attempts = queue.attempts;
if (sector > 1) {
CHECK(absl::IsInvalidArgument(
bycorf::spdk::sync::ReadDma(paths[0], buffer, 1)));
CHECK(absl::IsInvalidArgument(bycorf::spdk::sync::WriteDma(
paths[0], std::span(buffer).first(buffer.size() - 1), 0, false)));
}
CHECK(absl::IsInvalidArgument(
bycorf::spdk::sync::ReadDma(paths[0], buffer, capacity)));
const auto overflowing_offset =
std::numeric_limits<std::uint64_t>::max() - (sector - 1);
CHECK(absl::IsInvalidArgument(bycorf::spdk::sync::WriteDma(
paths[0], buffer, overflowing_offset, false)));
CHECK(queue.attempts == attempts);
bycorf::ReleaseSpdkStorageMetadataQpairs();
CHECK(qpairs.empty());
}

void CheckErrors(bycorf::Worker& worker, std::span<std::byte> buffer) {
auto& queue = *qpairs[0];
Completion oversized;
Expand Down Expand Up @@ -705,6 +801,51 @@ void RunCase(void (*check)(bycorf::Worker&, std::span<std::byte>)) {
// SPDK metadata and the allocator selection are process-global. Each child
// starts with fresh stubs and qpairs even when the binary runs every case or
// repeats them in shuffled order.
class SpdkSectorArithmeticDeathTest
: public testing::TestWithParam<std::uint32_t> {};

TEST_P(SpdkSectorArithmeticDeathTest, Async) {
const auto sector = GetParam();
ASSERT_EXIT(
{
for (auto& controller : controllers)
for (auto& ns : controller.namespaces) ns.sector_size = sector;
RunCase(CheckSectorArithmetic);
std::exit(0);
},
testing::ExitedWithCode(0), "");
}

TEST_P(SpdkSectorArithmeticDeathTest, Sync) {
const auto sector = GetParam();
ASSERT_EXIT(
{
controllers[0].namespaces[0].sector_size = sector;
CheckSyncSectorArithmetic();
std::exit(0);
},
testing::ExitedWithCode(0), "");
}

INSTANTIATE_TEST_SUITE_P(SectorSizes, SpdkSectorArithmeticDeathTest,
testing::Values(1, 512, 4096));

TEST(SpdkBackpressureDeathTest, InvalidSectorSize) {
ASSERT_EXIT(
{
CHECK(bycorf::ConfigureIoBackends({.spdk_storage = true}).ok());
for (const std::uint32_t sector : {0u, 520u, 0xffffffffu}) {
controllers[0].namespaces[0].sector_size = sector;
CHECK(absl::IsFailedPrecondition(
bycorf::ProbeSpdkStorage(paths[0]).status()));
}
bycorf::ReleaseSpdkStorageMetadataQpairs();
CHECK(qpairs.empty());
std::exit(0);
},
testing::ExitedWithCode(0), "");
}

TEST(SpdkBackpressureDeathTest, Queues) {
ASSERT_EXIT(
{
Expand Down