diff --git a/src/io/spdk_storage.cpp b/src/io/spdk_storage.cpp index 427f85f..4f616d1 100644 --- a/src/io/spdk_storage.cpp +++ b/src/io/spdk_storage.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -77,6 +78,9 @@ struct SpdkDevice { SpdkController* controller_ = nullptr; spdk_nvme_ns* ns_ = nullptr; std::uint32_t sector_size_ = 0; + // Discovery accepts only nonzero powers of two. Cache their exponent so + // request validation and LBA conversion need no variable integer division. + unsigned sector_shift_ = 0; std::uint64_t size_bytes_ = 0; }; @@ -220,6 +224,7 @@ absl::StatusOr GetDevice(std::string_view path) { device->controller_ = controller; device->ns_ = ns; device->sector_size_ = sector_size; + device->sector_shift_ = std::countr_zero(sector_size); device->size_bytes_ = spdk_nvme_ns_get_size(ns); SpdkDevice* raw = device.get(); g_spdk_devices.emplace(parsed->canonical_, std::move(device)); @@ -242,14 +247,14 @@ absl::Status SubmitSync(SpdkDevice& device, void* buffer, std::size_t bytes, if (bytes == 0) { return absl::OkStatus(); } - if (offset % device.sector_size_ != 0 || bytes % device.sector_size_ != 0 || + if (((offset | bytes) & (device.sector_size_ - 1)) != 0 || offset > device.size_bytes_ || bytes > device.size_bytes_ - offset) { return absl::Status(absl::StatusCode::kInvalidArgument, "unaligned or out-of-range SPDK I/O"); } - const std::uint64_t lba = offset / device.sector_size_; + const std::uint64_t lba = offset >> device.sector_shift_; const std::uint32_t lba_count = - static_cast(bytes / device.sector_size_); + static_cast(bytes >> device.sector_shift_); SpdkController& controller = *device.controller_; if (controller.sync_qpair_ == nullptr) { return absl::Status(absl::StatusCode::kFailedPrecondition, @@ -573,20 +578,22 @@ absl::Status SpdkBackend::SubmitIo(FixedFile file, void* buffer, } auto* device = static_cast(opened->device_); const std::uint32_t sector = device->sector_size_; + const unsigned sector_shift = device->sector_shift_; if (bytes > static_cast(std::numeric_limits::max()) || - offset % sector != 0 || bytes % sector != 0 || - bytes / sector > std::numeric_limits::max() || + ((offset | bytes) & (sector - 1)) != 0 || + (bytes >> sector_shift) > std::numeric_limits::max() || offset > device->size_bytes_ || bytes > device->size_bytes_ - offset) { return absl::Status(absl::StatusCode::kInvalidArgument, "invalid SPDK I/O range or alignment"); } - const PendingIo request{.tag_ = tag, - .buffer_ = buffer, - .lba_ = offset / sector, - .count_ = static_cast(bytes / sector), - .file_ = file, - .operation_ = write ? PendingIo::Operation::kWrite - : PendingIo::Operation::kRead}; + const PendingIo request{ + .tag_ = tag, + .buffer_ = buffer, + .lba_ = offset >> sector_shift, + .count_ = static_cast(bytes >> sector_shift), + .file_ = file, + .operation_ = + write ? PendingIo::Operation::kWrite : PendingIo::Operation::kRead}; tag->success_result_ = static_cast(bytes); // Reuse the lookup needed for validation on the ordinary submission path; // only a deferred retry needs to resolve the saved file index again. diff --git a/tests/spdk_backpressure_stub_test.cpp b/tests/spdk_backpressure_stub_test.cpp index a910d77..7f34839 100644 --- a/tests/spdk_backpressure_stub_test.cpp +++ b/tests/spdk_backpressure_stub_test.cpp @@ -52,6 +52,8 @@ namespace { enum class Operation { kRead, kWrite, kFlush }; struct Namespace { unsigned controller; + std::uint32_t sector_size = 512; + std::uint64_t size_bytes = 1ULL << 34; }; struct Controller { std::array namespaces; @@ -139,8 +141,12 @@ spdk_nvme_ns* __wrap_spdk_nvme_ctrlr_get_ns(spdk_nvme_ctrlr* ctrlr, &reinterpret_cast(ctrlr)->namespaces[id - 1]); } bool __wrap_spdk_nvme_ns_is_active(spdk_nvme_ns*) { return true; } -uint32_t __wrap_spdk_nvme_ns_get_sector_size(spdk_nvme_ns*) { return 512; } -uint64_t __wrap_spdk_nvme_ns_get_size(spdk_nvme_ns*) { return 1ULL << 34; } +uint32_t __wrap_spdk_nvme_ns_get_sector_size(spdk_nvme_ns* ns) { + return reinterpret_cast(ns)->sector_size; +} +uint64_t __wrap_spdk_nvme_ns_get_size(spdk_nvme_ns* ns) { + return reinterpret_cast(ns)->size_bytes; +} const spdk_nvme_ctrlr_data* __wrap_spdk_nvme_ctrlr_get_data(spdk_nvme_ctrlr*) { static const auto data = [] { spdk_nvme_ctrlr_data value{}; @@ -341,6 +347,96 @@ void CheckFixedBufferDescriptorLifetime(bycorf::Worker& worker, CHECK(saved_read.lba == 4 && saved_read.count == 1); } +void CheckSectorArithmetic(bycorf::Worker& worker, + std::span buffer) { + const auto sector = controllers[0].namespaces[0].sector_size; + const auto capacity = controllers[0].namespaces[0].size_bytes; + auto& queue = *qpairs[0]; + Completion tail, write, read; + CHECK(worker.SubmitRead({0}, buffer, capacity - buffer.size(), &tail).ok()); + CHECK(worker.SubmitWrite({1}, buffer, sector, &write).ok()); + CHECK(worker.SubmitRead({0}, buffer, 2ULL * sector, &read).ok()); + CHECK(queue.accepted.size() == 1); + CHECK(tail.calls + write.calls + read.calls == 0); + queue.completion_tokens = 3; + Drain(worker, [&] { return read.calls != 0; }); + CHECK(tail.result == static_cast(buffer.size())); + CHECK(write.result == static_cast(buffer.size())); + CHECK(read.result == static_cast(buffer.size())); + CHECK(queue.accepted.size() == 3); + CHECK(queue.accepted[0].lba == (capacity - buffer.size()) / sector); + CHECK(queue.accepted[1].operation == Operation::kWrite); + CHECK(queue.accepted[1].lba == 1 && queue.accepted[2].lba == 2); + for (const auto& command : queue.accepted) { + CHECK(command.buffer == buffer.data()); + CHECK(command.count == buffer.size() / sector); + } + + Completion rejected; + const auto attempts = queue.attempts; + if (sector > 1) { + CHECK( + absl::IsInvalidArgument(worker.SubmitRead({0}, buffer, 1, &rejected))); + CHECK(absl::IsInvalidArgument(worker.SubmitWrite( + {0}, buffer.first(buffer.size() - 1), 0, &rejected))); + } + CHECK(absl::IsInvalidArgument( + worker.SubmitRead({0}, buffer, capacity, &rejected))); + // Aligned offset whose addition with the request size would wrap. Bounds + // checks must continue to reject it before constructing a device command. + const auto overflowing_offset = + std::numeric_limits::max() - (sector - 1); + CHECK(absl::IsInvalidArgument( + worker.SubmitWrite({0}, buffer, overflowing_offset, &rejected))); + CHECK(absl::IsInvalidArgument(worker.SubmitReadFixed( + {0}, {.data_ = buffer.data(), .size_ = 0}, 0, &rejected))); + CHECK(absl::IsInvalidArgument(worker.SubmitReadFixed( + {0}, + {.data_ = buffer.data(), + .size_ = static_cast(std::numeric_limits::max()) + 1}, + 0, &rejected))); + CHECK(rejected.calls == 0 && queue.attempts == attempts); +} + +void CheckSyncSectorArithmetic() { + CHECK(bycorf::ConfigureIoBackends({.spdk_storage = true}).ok()); + CHECK(bycorf::ProbeSpdkStorage(paths[0]).ok()); + CHECK(qpairs.size() == 1); + auto& queue = *qpairs[0]; + const auto sector = controllers[0].namespaces[0].sector_size; + const auto capacity = controllers[0].namespaces[0].size_bytes; + // Link-time stubs do not DMA; direct metadata I/O needs no DMA allocator. + std::array buffer{}; + queue.completion_tokens = 2; + CHECK(bycorf::spdk::sync::ReadDma(paths[0], buffer, capacity - buffer.size()) + .ok()); + CHECK(bycorf::spdk::sync::WriteDma(paths[0], buffer, sector, false).ok()); + CHECK(queue.accepted.size() == 2); + CHECK(queue.accepted[0].lba == (capacity - buffer.size()) / sector); + CHECK(queue.accepted[1].operation == Operation::kWrite); + CHECK(queue.accepted[1].lba == 1); + for (const auto& command : queue.accepted) { + CHECK(command.buffer == buffer.data()); + CHECK(command.count == buffer.size() / sector); + } + const auto attempts = queue.attempts; + if (sector > 1) { + CHECK(absl::IsInvalidArgument( + bycorf::spdk::sync::ReadDma(paths[0], buffer, 1))); + CHECK(absl::IsInvalidArgument(bycorf::spdk::sync::WriteDma( + paths[0], std::span(buffer).first(buffer.size() - 1), 0, false))); + } + CHECK(absl::IsInvalidArgument( + bycorf::spdk::sync::ReadDma(paths[0], buffer, capacity))); + const auto overflowing_offset = + std::numeric_limits::max() - (sector - 1); + CHECK(absl::IsInvalidArgument(bycorf::spdk::sync::WriteDma( + paths[0], buffer, overflowing_offset, false))); + CHECK(queue.attempts == attempts); + bycorf::ReleaseSpdkStorageMetadataQpairs(); + CHECK(qpairs.empty()); +} + void CheckErrors(bycorf::Worker& worker, std::span buffer) { auto& queue = *qpairs[0]; Completion oversized; @@ -705,6 +801,51 @@ void RunCase(void (*check)(bycorf::Worker&, std::span)) { // SPDK metadata and the allocator selection are process-global. Each child // starts with fresh stubs and qpairs even when the binary runs every case or // repeats them in shuffled order. +class SpdkSectorArithmeticDeathTest + : public testing::TestWithParam {}; + +TEST_P(SpdkSectorArithmeticDeathTest, Async) { + const auto sector = GetParam(); + ASSERT_EXIT( + { + for (auto& controller : controllers) + for (auto& ns : controller.namespaces) ns.sector_size = sector; + RunCase(CheckSectorArithmetic); + std::exit(0); + }, + testing::ExitedWithCode(0), ""); +} + +TEST_P(SpdkSectorArithmeticDeathTest, Sync) { + const auto sector = GetParam(); + ASSERT_EXIT( + { + controllers[0].namespaces[0].sector_size = sector; + CheckSyncSectorArithmetic(); + std::exit(0); + }, + testing::ExitedWithCode(0), ""); +} + +INSTANTIATE_TEST_SUITE_P(SectorSizes, SpdkSectorArithmeticDeathTest, + testing::Values(1, 512, 4096)); + +TEST(SpdkBackpressureDeathTest, InvalidSectorSize) { + ASSERT_EXIT( + { + CHECK(bycorf::ConfigureIoBackends({.spdk_storage = true}).ok()); + for (const std::uint32_t sector : {0u, 520u, 0xffffffffu}) { + controllers[0].namespaces[0].sector_size = sector; + CHECK(absl::IsFailedPrecondition( + bycorf::ProbeSpdkStorage(paths[0]).status())); + } + bycorf::ReleaseSpdkStorageMetadataQpairs(); + CHECK(qpairs.empty()); + std::exit(0); + }, + testing::ExitedWithCode(0), ""); +} + TEST(SpdkBackpressureDeathTest, Queues) { ASSERT_EXIT( {