From c7ee77679080743718e59ce5bb33133eb7bcee12 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:14:49 +0000 Subject: [PATCH 01/14] lance-graph-dt64-lance: DR's map table in a separate Lance-versioned dataset Stores dt64's dr::MapTable (the versioned global -> local table, D-DT64-7 O5 / D10') in Lance, in a new excluded crate so dt64 stays free of lance and arrow: - one MapTable::advance is one Lance commit, written before the advance is applied, so a refused or failed commit leaves the table unchanged; - one row per local position, plus genesis, advance and removal markers; - the DR route version travels in the rows; the Lance dataset version is the commit counter, used to refuse a stale writer, detect a race, and read the table as of a past commit (restore_at, via checkout_version); - restore replays every advance at its own route version, including gaps from a shared clock, so every past version answers exactly as before; rows that do not form a log are refused. Frames are not restored: a frame from before a restore is foreign to the restored table. Executing at a past version after a restore is left open for the DR owner. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../2026-11-10-dt64-map-table-lance.md | 34 + .claude/board/entries/README.md | 3 +- ...-7-behaviour-addressing-registration-v1.md | 2 + Cargo.toml | 4 + crates/lance-graph-dt64-lance/Cargo.toml | 35 + crates/lance-graph-dt64-lance/src/lib.rs | 604 ++++++++++++++++++ .../lance-graph-dt64-lance/tests/map_log.rs | 196 ++++++ 7 files changed, 877 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-11-10-dt64-map-table-lance.md create mode 100644 crates/lance-graph-dt64-lance/Cargo.toml create mode 100644 crates/lance-graph-dt64-lance/src/lib.rs create mode 100644 crates/lance-graph-dt64-lance/tests/map_log.rs diff --git a/.claude/board/entries/2026-11-10-dt64-map-table-lance.md b/.claude/board/entries/2026-11-10-dt64-map-table-lance.md new file mode 100644 index 000000000..4e3c6d613 --- /dev/null +++ b/.claude/board/entries/2026-11-10-dt64-map-table-lance.md @@ -0,0 +1,34 @@ +# 2026-11-10 — DR map table stored as a separate Lance-versioned dataset + +**Status:** VERIFIED-IN-CODE (tests on a real Lance 13 dataset). Answers the +storage half of D-DT64-7 O5 / D10' (D-LITHO Q5: "which table"). + +**What.** `crates/lance-graph-dt64-lance` (excluded, own workspace, like +`lance-graph-dir-sim-lance`) stores `dr::MapTable` in Lance. dt64 stays free of +lance and arrow. + +- One `MapTable::advance` is one Lance commit. Rows are written first, then + the same advance is applied to the table; a refused or failed commit leaves + the table unchanged. +- One row per local position (D10'), plus genesis, advance and removal + markers. Ids only. +- Two clocks, never mixed: the DR route version is in the rows; the Lance + version is the physical commit counter (stale writer, race, and + `restore_at(k)` via `checkout_version`). +- `restore` replays every advance at its own route version, so an extension + stays an extension (generation and event counts equal), a replacement a new + generation, a removal an event. Route-version gaps from a shared clock + replay exactly. Rows that are not a log are refused. + +**Tests.** 4 unit + 6 integration. Disable runs, each restored: +stale-writer check removed → `a_stale_writer_writes_nothing` and +`create_refuses_an_existing_dataset` fail; gap ticks removed → +`route_versions_with_gaps_replay_as_route_versions` fails. + +**Open.** +- Frames are not restored: a frame is one table instance's witness, so a + frame from before a restore is `ForeignFrame` (pinned by a test). Executing + at a past version after a restore needs a frame issued at that version — + a DR owner decision. +- A table whose clock is shared with a `RouteLog` needs the route log + stored too; only the map is stored here. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 7d336cb7f..bcfc4ec41 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -591 entries, 2026-08-06 .. 2026-11-10. +592 entries, 2026-08-06 .. 2026-11-10. | date | entry id | finding | file | |---|---|---|---| | 2026-11-10 | `lab-resolve-borrow-all` | | [2026-11-10-lab-resolve-borrow-all.md](2026-11-10-lab-resolve-borrow-all.md) | +| 2026-11-10 | `dt64-map-table-lance` | | [2026-11-10-dt64-map-table-lance.md](2026-11-10-dt64-map-table-lance.md) | | 2026-11-10 | `dt64-lane-borrow-all` | | [2026-11-10-dt64-lane-borrow-all.md](2026-11-10-dt64-lane-borrow-all.md) | | 2026-11-10 | `dt64-int8-facet-stride` | | [2026-11-10-dt64-int8-facet-stride.md](2026-11-10-dt64-int8-facet-stride.md) | | 2026-11-10 | `D-DT64-GEMM-SHAPE-1` | | [2026-11-10-dt64-gemm-shape.md](2026-11-10-dt64-gemm-shape.md) | diff --git a/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md b/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md index 8c1b29771..d91dbf441 100644 --- a/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md +++ b/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md @@ -120,6 +120,8 @@ Preserved throughout: the `0xC401` name-bound reading, the `FnIndex` opcode cont **Status:** core shipped (`dr::MapTable`, T1–T4 green, both disable runs red). Stored selections wait on O4. +**Storage (2026-11-10):** the versioned table is stored as a separate Lance dataset by `crates/lance-graph-dt64-lance` (`LanceMapLog`). One `MapTable::advance` is one Lance commit; one row per local position; route versions in the rows, the Lance version only the commit counter; `restore` replays every advance exactly and `restore_at(k)` reads the table as of a past commit. OPEN: frames are not restored (a frame is one table instance's witness), so executing at a past version after a restore needs a DR decision; a table sharing its clock with a `RouteLog` needs the route log stored too. + O5 is a BindStore plan item (D-DT64-6 Amendment A). Litho is the mental model it came from, not a component. **What exists (VERIFIED-IN-CODE, `lance-graph-dt64/src/dr.rs`):** diff --git a/Cargo.toml b/Cargo.toml index 214702aa4..aa79c06ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,6 +176,10 @@ exclude = [ # (ogar-dir-core / ogar-ad / ogar-dir-sim). Verify via # `cargo test --manifest-path crates/lance-graph-dir-sim/Cargo.toml`. "crates/lance-graph-dir-sim", + # DR's versioned global -> local map table stored in Lance (D-DT64-7 + # O5). Own [workspace], lance + arrow; dt64 stays lance-free. Verify via + # `cargo test --manifest-path crates/lance-graph-dt64-lance/Cargo.toml`. + "crates/lance-graph-dt64-lance", # Cognitive Compilation loop (claude/cognitive-compilation-lance-graph-h8sgym). # The NEW piece is the Elixir-shaped template — thinking styles, JITson, and # i4-32D thinking-style vectors already exist; the template was the gap. Four diff --git a/crates/lance-graph-dt64-lance/Cargo.toml b/crates/lance-graph-dt64-lance/Cargo.toml new file mode 100644 index 000000000..6cb731a52 --- /dev/null +++ b/crates/lance-graph-dt64-lance/Cargo.toml @@ -0,0 +1,35 @@ +# lance-graph-dt64-lance — DR's versioned global -> local table (dt64 +# `dr::MapTable`) stored as a separate Lance-versioned dataset +# (D-DT64-7 O5 / D10', D-LITHO Q5). EXCLUDED from the lance-graph workspace +# (own [workspace] root), like lance-graph-dir-sim-lance: dt64 stays free of +# lance and arrow. +# +# One MapTable::advance is one Lance commit. The DR route version travels in +# the rows; the Lance dataset version is the physical commit counter, used to +# refuse a stale writer, to detect a race, and to read the table as of a past +# commit (`checkout_version`). +# +# Build/verify: +# cargo test --manifest-path crates/lance-graph-dt64-lance/Cargo.toml + +[package] +name = "lance-graph-dt64-lance" +version = "0.1.0" +edition = "2021" +publish = false +license = "Apache-2.0" +description = "DR's versioned global -> local map table stored in Lance: one advance per commit, replayed exactly at restore." + +[workspace] + +[dependencies] +lance-graph-dt64 = { path = "../lance-graph-dt64" } +# The lance family moves in lockstep with the lance-graph root (13 / arrow +# 58); a local or file URI is enough here. +lance = { version = "13.*", default-features = false } +arrow-array = "58" +arrow-schema = "58" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tempfile = "3" diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs new file mode 100644 index 000000000..346f75fdb --- /dev/null +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -0,0 +1,604 @@ +//! DR's versioned global → local table ([`MapTable`]) stored as a separate, +//! Lance-versioned dataset (D-DT64-7 O5 / D10', D-LITHO Q5). +//! +//! [`MapTable`] is the in-memory DR log: append-only, every past version +//! answerable, frames replaying the layout they were sealed under. This crate +//! makes that log durable without changing it. +//! +//! - **One advance is one Lance commit.** [`LanceMapLog::advance`] writes the +//! advance's rows in one commit, and only then applies the same advance to +//! the table. An advance is in the dataset whole or not at all. +//! - **One row per local position** (the D10' working model). A tile map at a +//! version is `width` plus its `local → global` rows in local order. A +//! removal and the advance itself are marker rows. Ids only, never values. +//! - **Two clocks, never mixed.** The DR route version travels in the rows. +//! The Lance dataset version is the physical commit counter: it refuses a +//! stale writer, detects a race, and names a past commit to read with +//! [`LanceMapLog::restore_at`]. It is never written into, or read as, a +//! route version. +//! - **Restore replays, exactly.** [`LanceMapLog::restore`] reads every row in +//! one scan and re-applies each advance to a fresh table at the same route +//! version, so every version answers as before: an extension stays an +//! extension, a replacement a new generation, a removal an event. Rows that +//! do not form a valid log are refused; nothing is repaired. +//! +//! What a restore does **not** carry: frames. A [`Frame`] is a witness issued +//! by one table instance, so frames sealed before a restore are foreign to +//! the restored table (`Stale(ForeignFrame)`). Past versions stay readable +//! through [`MapTable::diagnose_at`]; executing at a past version after a +//! restore would need a frame issued at that version, which DR does not +//! offer (OPEN, DR owner). +//! +//! [`Frame`]: lance_graph_dt64::dr::Frame + +use std::sync::Arc; + +use arrow_array::{ + Array, RecordBatch, RecordBatchIterator, UInt16Array, UInt32Array, UInt64Array, UInt8Array, +}; +use arrow_schema::{ArrowError, DataType, Field, Schema}; +use lance::dataset::{WriteMode, WriteParams}; +use lance::Dataset; +use lance_graph_dt64::dr::{ + LocalMap, MapRefusal, MapTable, RouteVersion, TileId, VersionClock, Width, +}; + +/// Row kinds. +const GENESIS: u8 = 0; +const ADVANCE: u8 = 1; +const POSITION: u8 = 2; +const REMOVED: u8 = 3; + +/// One stored row. `source` / `relation_map` are set on genesis and advance +/// rows; `tile`, `width`, `local`, `global` on position rows (`tile` also on +/// removals); unused fields are zero. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Row { + version: u64, + kind: u8, + source: u64, + relation_map: u64, + tile: u32, + width: u16, + local: u16, + global: u64, +} + +impl Row { + fn marker(version: u64, kind: u8, source: u64, relation_map: u64) -> Row { + Row { + version, + kind, + source, + relation_map, + tile: 0, + width: 0, + local: 0, + global: 0, + } + } +} + +/// Why a save or a restore failed. +#[derive(Debug)] +pub enum StoreError { + /// Lance refused an operation. + Lance(lance::Error), + /// A stored column is missing, has another type, or holds a null. + Schema(String), + /// The stored rows do not form a valid log. + Replay(ReplayError), + /// One advance named the same tile twice. Nothing was written. + DuplicateTile(TileId), + /// The dataset is not at the version this writer last saw (`None`: no + /// dataset). Nothing was written. + StaleWriter { + expected: Option, + found: Option, + }, + /// The commit landed, but not as the next dataset version: another + /// writer committed between this writer's check and its commit. The rows + /// are stored; the table was not advanced. + Race { expected: u64, found: u64 }, + /// The rows were stored at route version `stored`, but the table's clock + /// had moved, so the advance was issued at `issued`. Happens only when a + /// clone of the table's clock ticked between the check and the advance. + ClockMoved { stored: u64, issued: u64 }, +} + +impl From for StoreError { + fn from(e: lance::Error) -> Self { + StoreError::Lance(e) + } +} + +/// Why stored rows are not a valid log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReplayError { + /// The dataset holds no rows. + Empty, + /// The first row is not the genesis row, or a second genesis follows. + Genesis, + /// A row's route version is not after the previous advance's. + VersionOrder { previous: u64, found: u64 }, + /// A version's rows do not start with its advance marker. + MissingAdvance { version: u64 }, + /// A row kind this crate does not write. + UnknownKind(u8), + /// A width other than 64 or 256. + Width(u16), + /// A tile's positions are not `0..len` in order, or mix widths. + Positions { version: u64, tile: u32 }, + /// A tile appears twice in one advance. + DuplicateTile { version: u64, tile: u32 }, + /// The positions do not form a map (duplicate id, too many). + Map { + version: u64, + tile: u32, + refusal: MapRefusal, + }, + /// Re-applying the advance issued another route version. + Reissued { stored: u64, issued: u64 }, +} + +fn schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("version", DataType::UInt64, false), + Field::new("kind", DataType::UInt8, false), + Field::new("source", DataType::UInt64, false), + Field::new("relation_map", DataType::UInt64, false), + Field::new("tile", DataType::UInt32, false), + Field::new("width", DataType::UInt16, false), + Field::new("local", DataType::UInt16, false), + Field::new("global", DataType::UInt64, false), + ])) +} + +fn to_batch(rows: &[Row]) -> Result { + RecordBatch::try_new( + schema(), + vec![ + Arc::new(UInt64Array::from_iter_values( + rows.iter().map(|r| r.version), + )), + Arc::new(UInt8Array::from_iter_values(rows.iter().map(|r| r.kind))), + Arc::new(UInt64Array::from_iter_values(rows.iter().map(|r| r.source))), + Arc::new(UInt64Array::from_iter_values( + rows.iter().map(|r| r.relation_map), + )), + Arc::new(UInt32Array::from_iter_values(rows.iter().map(|r| r.tile))), + Arc::new(UInt16Array::from_iter_values(rows.iter().map(|r| r.width))), + Arc::new(UInt16Array::from_iter_values(rows.iter().map(|r| r.local))), + Arc::new(UInt64Array::from_iter_values(rows.iter().map(|r| r.global))), + ], + ) +} + +fn column<'a, T: 'static + Array>(b: &'a RecordBatch, name: &str) -> Result<&'a T, StoreError> { + let c = b + .column_by_name(name) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| StoreError::Schema(format!("column `{name}`")))?; + if c.null_count() != 0 { + return Err(StoreError::Schema(format!("null in `{name}`"))); + } + Ok(c) +} + +fn from_batch(b: &RecordBatch) -> Result, StoreError> { + let version: &UInt64Array = column(b, "version")?; + let kind: &UInt8Array = column(b, "kind")?; + let source: &UInt64Array = column(b, "source")?; + let relation_map: &UInt64Array = column(b, "relation_map")?; + let tile: &UInt32Array = column(b, "tile")?; + let width: &UInt16Array = column(b, "width")?; + let local: &UInt16Array = column(b, "local")?; + let global: &UInt64Array = column(b, "global")?; + Ok((0..b.num_rows()) + .map(|i| Row { + version: version.value(i), + kind: kind.value(i), + source: source.value(i), + relation_map: relation_map.value(i), + tile: tile.value(i), + width: width.value(i), + local: local.value(i), + global: global.value(i), + }) + .collect()) +} + +fn encode_width(w: Width) -> u16 { + match w { + Width::W64 => 64, + Width::W256 => 256, + } +} + +fn decode_width(w: u16) -> Result { + match w { + 64 => Ok(Width::W64), + 256 => Ok(Width::W256), + other => Err(ReplayError::Width(other)), + } +} + +/// The rows of one advance at `version`: its marker, then each change. +fn advance_rows( + version: u64, + source: u64, + relation_map: u64, + changes: &[(TileId, Option)], +) -> Result, StoreError> { + let mut seen: Vec = changes.iter().map(|(t, _)| *t).collect(); + seen.sort_unstable(); + if let Some(w) = seen.windows(2).find(|w| w[0] == w[1]) { + return Err(StoreError::DuplicateTile(w[0])); + } + let mut rows = vec![Row::marker(version, ADVANCE, source, relation_map)]; + for (tile, map) in changes { + match map { + None => rows.push(Row { + tile: *tile, + ..Row::marker(version, REMOVED, 0, 0) + }), + Some(m) => { + let width = encode_width(m.width()); + for (local, &global) in m.view().globals().iter().enumerate() { + rows.push(Row { + tile: *tile, + width, + // A map holds at most 256 positions. + local: local as u16, + global, + ..Row::marker(version, POSITION, 0, 0) + }); + } + if m.is_empty() { + // An empty map still replaces the tile: keep it visible. + rows.push(Row { + tile: *tile, + width, + local: u16::MAX, + ..Row::marker(version, POSITION, 0, 0) + }); + } + } + } + } + Ok(rows) +} + +/// Rebuild a table from its rows, re-applying every advance at its own route +/// version. +fn replay(rows: &[Row]) -> Result { + let (genesis, rest) = rows.split_first().ok_or(ReplayError::Empty)?; + if genesis.kind != GENESIS { + return Err(ReplayError::Genesis); + } + let clock = VersionClock::new(); + while clock.current().0 < genesis.version { + clock.tick(); + } + let mut table = MapTable::with_clock(genesis.source, genesis.relation_map, clock.clone()); + let mut i = 0; + let mut previous = genesis.version; + while i < rest.len() { + let marker = rest[i]; + match marker.kind { + GENESIS => return Err(ReplayError::Genesis), + ADVANCE => {} + POSITION | REMOVED => { + return Err(ReplayError::MissingAdvance { + version: marker.version, + }) + } + other => return Err(ReplayError::UnknownKind(other)), + } + let version = marker.version; + if version <= previous { + return Err(ReplayError::VersionOrder { + previous, + found: version, + }); + } + let end = i + + 1 + + rest[i + 1..] + .iter() + .take_while(|r| r.version == version && r.kind != ADVANCE) + .count(); + let changes = changes_of(version, &rest[i + 1..end])?; + while clock.current().0 + 1 < version { + clock.tick(); + } + let issued = table.advance(marker.source, marker.relation_map, changes); + if issued.0 != version { + return Err(ReplayError::Reissued { + stored: version, + issued: issued.0, + }); + } + previous = version; + i = end; + } + Ok(table) +} + +/// The changes of one advance from its position and removal rows. +fn changes_of(version: u64, rows: &[Row]) -> Result)>, ReplayError> { + let mut changes: Vec<(TileId, Option)> = Vec::new(); + let mut j = 0; + while j < rows.len() { + let tile = rows[j].tile; + if changes.iter().any(|(t, _)| *t == tile) { + return Err(ReplayError::DuplicateTile { version, tile }); + } + match rows[j].kind { + REMOVED => { + changes.push((tile, None)); + j += 1; + } + POSITION => { + let run = rows[j..] + .iter() + .take_while(|r| r.kind == POSITION && r.tile == tile) + .count(); + let span = &rows[j..j + run]; + let width = decode_width(span[0].width)?; + let empty = run == 1 && span[0].local == u16::MAX; + let ordered = span + .iter() + .enumerate() + .all(|(k, r)| usize::from(r.local) == k && r.width == span[0].width); + if !empty && !ordered { + return Err(ReplayError::Positions { version, tile }); + } + let globals: Vec = if empty { + Vec::new() + } else { + span.iter().map(|r| r.global).collect() + }; + let map = + LocalMap::prepare(width, &globals).map_err(|refusal| ReplayError::Map { + version, + tile, + refusal, + })?; + changes.push((tile, Some(map))); + j += run; + } + other => return Err(ReplayError::UnknownKind(other)), + } + } + Ok(changes) +} + +fn is_not_found(e: &lance::Error) -> bool { + matches!(e, lance::Error::DatasetNotFound { .. }) +} + +/// The dataset version a commit on top of `seen` makes: Lance numbers a new +/// dataset 1 and each later commit one higher. +fn next_version(seen: Option) -> u64 { + seen.map_or(1, |v| v + 1) +} + +/// A writer's handle on one DR map table's dataset at `uri`. +#[derive(Debug, Clone)] +pub struct LanceMapLog { + uri: String, + /// The dataset version this writer last saw. + version: Option, +} + +impl LanceMapLog { + /// Create the dataset with its genesis row and return a new table with + /// the given revisions and its writer. Refuses if a dataset is already at + /// `uri`. + pub async fn create( + uri: impl Into, + source: u64, + relation_map: u64, + ) -> Result<(MapTable, LanceMapLog), StoreError> { + let mut log = LanceMapLog { + uri: uri.into(), + version: None, + }; + let table = MapTable::new(source, relation_map); + let genesis = Row::marker(table.current_version().0, GENESIS, source, relation_map); + log.commit(&[genesis]).await?; + Ok((table, log)) + } + + /// The dataset version this writer last saw (the physical commit + /// counter, not a route version). + pub fn dataset_version(&self) -> Option { + self.version + } + + /// Store one advance as one Lance commit, then apply it to `table`. + /// Returns the route version it was issued at. + /// + /// # Errors + /// + /// - [`StoreError::DuplicateTile`] or [`StoreError::StaleWriter`]: + /// nothing is written and the table is unchanged. + /// - [`StoreError::Race`]: the rows landed as another dataset version; + /// the table is unchanged. + /// - [`StoreError::ClockMoved`]: see the variant. + pub async fn advance( + &mut self, + table: &mut MapTable, + source: u64, + relation_map: u64, + changes: Vec<(TileId, Option)>, + ) -> Result { + let stored = table.current_version().0 + 1; + let rows = advance_rows(stored, source, relation_map, &changes)?; + self.commit(&rows).await?; + let issued = table.advance(source, relation_map, changes); + if issued.0 != stored { + return Err(StoreError::ClockMoved { + stored, + issued: issued.0, + }); + } + Ok(issued) + } + + async fn commit(&mut self, rows: &[Row]) -> Result { + let found = match Dataset::open(&self.uri).await { + Ok(ds) => Some(ds.latest_version_id().await?), + Err(e) if is_not_found(&e) => None, + Err(e) => return Err(e.into()), + }; + if found != self.version { + return Err(StoreError::StaleWriter { + expected: self.version, + found, + }); + } + let rb = to_batch(rows).map_err(|e| StoreError::Schema(e.to_string()))?; + let reader = RecordBatchIterator::new([Ok(rb)], schema()); + let mode = if found.is_some() { + WriteMode::Append + } else { + WriteMode::Create + }; + let params = WriteParams { + mode, + ..WriteParams::default() + }; + let ds = match Dataset::write(reader, self.uri.as_str(), Some(params)).await { + Ok(ds) => ds, + Err(lance::Error::DatasetAlreadyExists { .. }) => { + return Err(StoreError::StaleWriter { + expected: None, + found: Dataset::open(&self.uri) + .await + .ok() + .map(|d| d.version().version), + }) + } + Err(e) => return Err(e.into()), + }; + let made = ds.version().version; + let next = next_version(found); + if made != next { + return Err(StoreError::Race { + expected: next, + found: made, + }); + } + self.version = Some(made); + Ok(made) + } + + /// Read the whole table at `uri` once and replay it, with a writer at the + /// dataset's current version. + pub async fn restore(uri: impl Into) -> Result<(MapTable, LanceMapLog), StoreError> { + let uri = uri.into(); + let ds = Dataset::open(&uri).await?; + let table = read(&ds).await?; + let version = Some(ds.version().version); + Ok((table, LanceMapLog { uri, version })) + } + + /// The table as of dataset version `dataset_version` (a past commit), + /// read through Lance's own version history. Read-only: no writer is + /// returned, since appending on top of a past commit would fork the log. + pub async fn restore_at( + uri: impl Into, + dataset_version: u64, + ) -> Result { + let uri = uri.into(); + let ds = Dataset::open(&uri) + .await? + .checkout_version(dataset_version) + .await?; + read(&ds).await + } +} + +async fn read(ds: &Dataset) -> Result { + let all = ds.scan().try_into_batch().await?; + let rows = from_batch(&all)?; + replay(&rows).map_err(StoreError::Replay) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(w: Width, ids: &[u64]) -> Option { + Some(LocalMap::prepare(w, ids).unwrap()) + } + + #[test] + fn rows_round_trip_through_a_record_batch() { + let rows = advance_rows(7, 3, 4, &[(1, map(Width::W64, &[9, 8])), (2, None)]).unwrap(); + assert_eq!(rows.len(), 4); + assert_eq!(from_batch(&to_batch(&rows).unwrap()).unwrap(), rows); + } + + #[test] + fn an_advance_naming_a_tile_twice_is_refused() { + let err = advance_rows(1, 1, 1, &[(5, None), (5, map(Width::W64, &[1]))]).unwrap_err(); + assert!(matches!(err, StoreError::DuplicateTile(5))); + } + + #[test] + fn replay_refuses_rows_that_are_not_a_log() { + let genesis = Row::marker(0, GENESIS, 1, 1); + let good = advance_rows(1, 1, 1, &[(1, map(Width::W64, &[4, 5]))]).unwrap(); + let mut rows = vec![genesis]; + rows.extend(&good); + assert!(replay(&rows).is_ok()); + + assert_eq!(replay(&rows[1..]).unwrap_err(), ReplayError::Genesis); + assert_eq!(replay(&[]).unwrap_err(), ReplayError::Empty); + + let mut swapped = rows.clone(); + swapped.swap(2, 3); + assert_eq!( + replay(&swapped).unwrap_err(), + ReplayError::Positions { + version: 1, + tile: 1 + } + ); + + let mut dup = rows.clone(); + dup[3].global = dup[2].global; + assert!(matches!( + replay(&dup).unwrap_err(), + ReplayError::Map { + refusal: MapRefusal::Duplicate, + .. + } + )); + + let mut back = rows.clone(); + back.extend(advance_rows(1, 1, 1, &[]).unwrap()); + assert_eq!( + replay(&back).unwrap_err(), + ReplayError::VersionOrder { + previous: 1, + found: 1 + } + ); + + let orphan = vec![genesis, good[1]]; + assert_eq!( + replay(&orphan).unwrap_err(), + ReplayError::MissingAdvance { version: 1 } + ); + } + + #[test] + fn the_next_version_is_one_more_or_one_for_a_new_dataset() { + assert_eq!(next_version(None), 1); + assert_eq!(next_version(Some(4)), 5); + } +} diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs new file mode 100644 index 000000000..fbe3b0316 --- /dev/null +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -0,0 +1,196 @@ +//! A DR map table survives a round trip through its Lance dataset exactly: +//! every past route version answers as before, one advance is one commit, +//! a past commit reads the table as of then, a stale writer is refused, and +//! route versions are never confused with dataset versions. + +use lance_graph_dt64::dr::{FrameRefusal, LocalMap, MapRead, MapTable, RouteVersion, Width}; +use lance_graph_dt64_lance::{LanceMapLog, StoreError}; + +const TILES: [u32; 3] = [1, 2, 3]; + +fn map(w: Width, ids: &[u64]) -> Option { + Some(LocalMap::prepare(w, ids).unwrap()) +} + +/// What a table answers for `tile` at `version`: `(width, globals)`, or the +/// refusal. +fn answer(t: &MapTable, version: u64, tile: u32) -> Result<(u16, Vec), MapRead> { + let d = t.diagnose_at(RouteVersion(version), tile)?; + let width = match d.width() { + Width::W64 => 64, + Width::W256 => 256, + }; + let globals = (0..d.len()) + .map(|l| d.global(u8::try_from(l).unwrap()).unwrap()) + .collect(); + Ok((width, globals)) +} + +/// Every version up to `upto`, every tile: the two tables answer alike. +fn assert_same(a: &MapTable, b: &MapTable, upto: u64) { + for v in 0..=upto { + for tile in TILES { + assert_eq!( + answer(a, v, tile), + answer(b, v, tile), + "version {v} tile {tile}" + ); + } + } +} + +/// Six advances covering every kind of history event. +async fn history(log: &mut LanceMapLog, t: &mut MapTable) { + let steps: Vec)>> = vec![ + vec![ + (1, map(Width::W64, &[10, 11, 12])), + (2, map(Width::W256, &[7, 3, 9, 1])), + ], + vec![(1, map(Width::W64, &[10, 11, 12, 13]))], // extension + vec![(1, map(Width::W64, &[13, 10, 11, 12]))], // replacement (reorder) + vec![(2, None)], // removal + vec![], // advance with no change + vec![(2, map(Width::W64, &[7])), (3, map(Width::W256, &[]))], // re-add, empty map + ]; + for (k, changes) in steps.into_iter().enumerate() { + let v = log.advance(t, 1, 1 + k as u64, changes).await.unwrap(); + assert_eq!(v, RouteVersion(k as u64 + 1)); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_restored_table_answers_every_past_version_exactly() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); + history(&mut log, &mut t).await; + + let (r, rlog) = LanceMapLog::restore(&uri).await.unwrap(); + assert_eq!(r.current_version(), t.current_version()); + assert_eq!(r.revisions(), t.revisions()); + assert_same(&t, &r, t.current_version().0); + for tile in TILES { + assert_eq!( + r.generation_count(tile), + t.generation_count(tile), + "tile {tile}" + ); + assert_eq!(r.event_count(tile), t.event_count(tile), "tile {tile}"); + } + // The extension is stored as an extension: tile 1 has two generations + // (the original, extended in place, and the reorder), not three. + assert_eq!(r.generation_count(1), 2); + // Genesis plus six advances: seven commits. + assert_eq!(log.dataset_version(), Some(7)); + assert_eq!(rlog.dataset_version(), Some(7)); +} + +/// Lance's own version history is the DR history: dataset version `k` holds +/// genesis and the first `k - 1` advances, and answers each of them as the +/// live table does. +#[tokio::test(flavor = "multi_thread")] +async fn each_past_commit_reads_the_table_as_of_then() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); + history(&mut log, &mut t).await; + for k in 1..=7u64 { + let past = LanceMapLog::restore_at(&uri, k).await.unwrap(); + assert_eq!( + past.current_version(), + RouteVersion(k - 1), + "dataset version {k}" + ); + assert_same(&t, &past, k - 1); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_stale_writer_writes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); + let (mut t2, mut log2) = LanceMapLog::restore(&uri).await.unwrap(); + log.advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[1, 2]))]) + .await + .unwrap(); + + let before = t2.current_version(); + let err = log2 + .advance(&mut t2, 1, 1, vec![(1, map(Width::W64, &[5]))]) + .await + .unwrap_err(); + assert!(matches!( + err, + StoreError::StaleWriter { + expected: Some(1), + found: Some(2) + } + )); + assert_eq!(t2.current_version(), before, "the table was not advanced"); + let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); + assert_eq!( + answer(&r, 1, 1), + Ok((64, vec![1, 2])), + "only the first writer's advance is stored" + ); +} + +/// A table whose clock is shared (here: ticked from a clone, as a RouteLog +/// sharing it would) issues route versions with gaps. They are stored and +/// replayed as route versions, independent of the dataset version. +#[tokio::test(flavor = "multi_thread")] +async fn route_versions_with_gaps_replay_as_route_versions() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); + let shared = t.clock(); + for _ in 0..5 { + shared.tick(); + } + let v = log + .advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[42]))]) + .await + .unwrap(); + assert_eq!(v, RouteVersion(6)); + assert_eq!(log.dataset_version(), Some(2), "two commits, not six"); + + let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); + assert_eq!(r.current_version(), RouteVersion(6)); + assert_eq!(answer(&r, 5, 1), Err(MapRead::NoSuchTile(1))); + assert_eq!(answer(&r, 6, 1), Ok((64, vec![42]))); +} + +/// Frames are witnesses of one table instance, so a frame sealed before a +/// restore is foreign to the restored table. +#[tokio::test(flavor = "multi_thread")] +async fn a_frame_from_before_a_restore_is_foreign() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); + log.advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[1]))]) + .await + .unwrap(); + let frame = t.seal(); + assert!(t.map(&frame, 1).is_ok()); + let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); + assert!(matches!( + r.map(&frame, 1), + Err(MapRead::Stale(FrameRefusal::ForeignFrame)) + )); + assert!(r.map(&r.seal(), 1).is_ok()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn create_refuses_an_existing_dataset() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); + LanceMapLog::create(&uri, 1, 1).await.unwrap(); + assert!(matches!( + LanceMapLog::create(&uri, 1, 1).await.unwrap_err(), + StoreError::StaleWriter { + expected: None, + found: Some(1) + } + )); +} From 050fa942d0802362975d59d4f36c739136d93200 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:15:16 +0000 Subject: [PATCH 02/14] ci: build and test lance-graph-dt64-lance Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .github/workflows/dt64-map-table-lance.yml | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/dt64-map-table-lance.yml diff --git a/.github/workflows/dt64-map-table-lance.yml b/.github/workflows/dt64-map-table-lance.yml new file mode 100644 index 000000000..62f29aeb6 --- /dev/null +++ b/.github/workflows/dt64-map-table-lance.yml @@ -0,0 +1,70 @@ +name: dt64 map table in Lance +# lance-graph-dt64-lance stores dt64's DR map table (dr::MapTable) in Lance. +# It is workspace-excluded (own [workspace] root), so no other job builds it. +# Without this one a change to dt64's dr module or a lance bump could break it +# unnoticed. +# +# lance's build compiles .proto files: `protoc` alone is not enough, the +# well-known includes (libprotobuf-dev) are needed too. +on: + push: + branches: + - main + pull_request: + paths: + - crates/lance-graph-contract/** + - crates/lance-graph-dt64/** + - crates/lance-graph-dt64-lance/** + - .github/workflows/dt64-map-table-lance.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + RUST_BACKTRACE: "1" + +jobs: + map-table: + name: DR map table in Lance + runs-on: ubuntu-24.04 + timeout-minutes: 45 + defaults: + run: + working-directory: lance-graph + steps: + - uses: actions/checkout@v4 + with: + path: lance-graph + persist-credentials: false + - name: Install protoc and the protobuf includes (for lance) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev + - name: Setup rust toolchain (pinned by rust-toolchain.toml) + run: rustup show + # See dir-sim-persist.yml: spell target-cpu=native out as this runner's + # feature list so the cache never reuses an artifact on another CPU. + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + - uses: Swatinem/rust-cache@v2 + with: + key: cpu-${{ steps.cpu.outputs.flags }} + workspaces: lance-graph/crates/lance-graph-dt64-lance + - name: fmt + run: cargo fmt --manifest-path crates/lance-graph-dt64-lance/Cargo.toml --check + - name: clippy + run: cargo clippy --manifest-path crates/lance-graph-dt64-lance/Cargo.toml --all-targets -- -D warnings + - name: test + run: cargo test --manifest-path crates/lance-graph-dt64-lance/Cargo.toml From 52dfc8cbf77d0390881cd08bc0193086d3090f45 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:58:55 +0000 Subject: [PATCH 03/14] wal_shootout: a dr64 mode for DR/DT64 64x64 advances A DR64 tile map is 64 global ids x 8 bytes, one 512-byte payload, so a DR advance is the shoot-out's SoA-512 cycle with one cast per changed tile. The existing sizes start at N = 64; dr64 runs the same loop at N = 1, 4, 16 and 64 changed tiles per advance, both arms, to pick the WAL for the DR map table (#1993). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph/examples/wal_shootout.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/lance-graph/examples/wal_shootout.rs b/crates/lance-graph/examples/wal_shootout.rs index a229936c4..00f003b48 100644 --- a/crates/lance-graph/examples/wal_shootout.rs +++ b/crates/lance-graph/examples/wal_shootout.rs @@ -4,6 +4,8 @@ //! ```text //! # cost table (D-WAL-2) //! cargo run --release -p lance-graph --features lancedb-sdk --example wal_shootout -- latency +//! # DR/DT64 64x64: N = 1, 4, 16, 64 changed tiles per advance +//! cargo run --release -p lance-graph --features lancedb-sdk --example wal_shootout -- dr64 //! # soak, one process per arm (D-WAL-3) //! cargo run --release -p lance-graph --features lancedb-sdk --example wal_shootout -- \ //! soak @@ -101,13 +103,21 @@ impl Arm { } } -async fn latency(root: &str) { +/// The SoA-512 sizes the shoot-out was run at (D-WAL-2). +const SOA512: [(usize, u64); 3] = [(64, 200), (1024, 200), (8192, 40)]; + +/// DR/DT64 64x64: one cast per changed DR64 tile (64 global ids x 8 bytes = +/// one 512-byte payload). N is the number of tiles one advance changes: a +/// single tile up to the whole 64-tile cube. +const DR64: [(usize, u64); 4] = [(1, 200), (4, 200), (16, 200), (64, 200)]; + +async fn latency(root: &str, sizes: &[(usize, u64)]) { println!( "tier: avx512f={} | host I/O bound: numbers are per cycle (µs) and per stored row (ns)", cfg!(target_feature = "avx512f") ); println!("arm N cycles p50_us p99_us ns/row rows MiB files versions scan_ms"); - for (n, cycles) in [(64usize, 200u64), (1024, 200), (8192, 40)] { + for &(n, cycles) in sizes { for kind in ["L", "M10", "M100"] { let dir = format!("{root}/lat-{kind}-{n}"); let _ = std::fs::remove_dir_all(&dir); @@ -197,7 +207,8 @@ async fn soak(kind: &str, dir: &str, minutes: u64, n: usize, interval_ms: u64, c async fn main() { let args: Vec = std::env::args().collect(); match args.get(1).map(String::as_str) { - Some("latency") => latency(args.get(2).expect("scratch dir")).await, + Some("latency") => latency(args.get(2).expect("scratch dir"), &SOA512).await, + Some("dr64") => latency(args.get(2).expect("scratch dir"), &DR64).await, Some("soak") => { soak( &args[2], From c6fd03073c17e66d62ba455cbb972b562cf2453f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:10:17 +0000 Subject: [PATCH 04/14] dt64: claim a route version now, apply the advance at it later MapTable::claim takes the next version off the (possibly shared) clock; MapTable::advance_at applies an advance at exactly that version once its rows are durable, even if other clock holders ticked in between. It refuses, changing nothing, a version at or before the last applied one and a version the clock never issued. advance() is now claim + apply, so its behaviour is unchanged. This is what lets a WAL-backed map log carry the route version on the cast instead of re-deriving it after the seal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64/src/dr.rs | 62 +++++++++++++++++++++++++- crates/lance-graph-dt64/tests/dr.rs | 68 ++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 3 deletions(-) diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index 47cfe9dd2..c746907ac 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -659,6 +659,21 @@ fn fnv_ids(globals: &[u64]) -> u64 { /// A tile's name in a [`MapTable`], chosen by the caller. pub type TileId = u32; +/// Why [`MapTable::advance_at`] refused a claimed version. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdvanceRefusal { + /// `version` is at or before `last`, the last version this table applied. + NotAfterLast { + last: RouteVersion, + version: RouteVersion, + }, + /// The clock has not issued `version` (its last issued is `current`). + NotIssued { + version: RouteVersion, + current: RouteVersion, + }, +} + /// Why a [`MapTable`] read was refused. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MapRead { @@ -916,6 +931,52 @@ impl MapTable { changes: Vec<(TileId, Option)>, ) -> RouteVersion { let version = self.frames.clock.tick(); + self.apply(version, source, relation_map, changes); + version + } + + /// Take the next version from the (possibly shared) clock for an advance + /// that will be applied later with [`MapTable::advance_at`], once its + /// rows are durable. A claimed version that is never applied is a gap: + /// no advance, no frame, nothing to replay. + pub fn claim(&self) -> RouteVersion { + self.frames.clock.tick() + } + + /// Apply an advance at a version claimed earlier with [`MapTable::claim`], + /// exactly as [`MapTable::advance`] would have at that version. Other + /// holders of the clock may have ticked since the claim; the advance still + /// lands at `version`. + /// + /// Refuses, changing nothing, a version at or before the last one this + /// table applied (its event logs would leave version order) and a version + /// the clock has not issued. + pub fn advance_at( + &mut self, + version: RouteVersion, + source: u64, + relation_map: u64, + changes: Vec<(TileId, Option)>, + ) -> Result { + let current = self.frames.clock.current(); + if version > current { + return Err(AdvanceRefusal::NotIssued { version, current }); + } + let last = self.frames.version; + if version <= last { + return Err(AdvanceRefusal::NotAfterLast { last, version }); + } + self.apply(version, source, relation_map, changes); + Ok(version) + } + + fn apply( + &mut self, + version: RouteVersion, + source: u64, + relation_map: u64, + changes: Vec<(TileId, Option)>, + ) { for (tile, map) in changes { match map { Some(m) => self.tiles.entry(tile).or_default().append(version, m), @@ -935,7 +996,6 @@ impl MapTable { }, version, ); - version } /// Seal the current frame. diff --git a/crates/lance-graph-dt64/tests/dr.rs b/crates/lance-graph-dt64/tests/dr.rs index cea3036f8..8aea1ef22 100644 --- a/crates/lance-graph-dt64/tests/dr.rs +++ b/crates/lance-graph-dt64/tests/dr.rs @@ -344,8 +344,8 @@ mod locate { mod vmap { use super::map; use lance_graph_dt64::dr::{ - Axis, FrameRefusal, MapRead, MapRefusal, MapTable, RouteLog, RouteVersion, VersionClock, - Width, + AdvanceRefusal, Axis, FrameRefusal, MapRead, MapRefusal, MapTable, RouteLog, RouteVersion, + VersionClock, Width, }; use std::alloc::{GlobalAlloc, Layout, System}; use std::cell::Cell; @@ -553,6 +553,70 @@ mod vmap { assert!(copy.current_version() > log.current_version()); } + /// A version claimed before a write is applied exactly, even when another + /// holder of the shared clock ticks between the claim and the apply (a + /// cast is sealed by the WAL while a RouteLog keeps routing). Fails if the + /// apply takes a fresh tick instead of the claimed version. + #[test] + fn a_claimed_version_is_applied_exactly_after_the_clock_moves() { + let clock = VersionClock::new(); + let mut log = RouteLog::with_clock(clock.clone()); + let mut t = MapTable::with_clock(1, 1, clock.clone()); + let id = Axis::new(Width::W64, 1); + let v = t.claim(); + let moved = log.route(3, 9, id).unwrap(); + assert!( + moved > v, + "anti-vacuity: the clock really moved past the claim" + ); + assert_eq!( + t.advance_at(v, 1, 1, vec![(1, Some(map(&[10, 11])))]), + Ok(v) + ); + assert_eq!(t.seal().version(), v); + let d = t.diagnose_at(v, 1).unwrap(); + assert_eq!((d.len(), d.global(0), d.global(1)), (2, Some(10), Some(11))); + assert_eq!( + t.diagnose_at(RouteVersion(v.0 - 1), 1).unwrap_err(), + MapRead::NoSuchTile(1) + ); + // Silence twin: plain advance still takes the next tick. + let w = t.advance(1, 1, vec![(1, Some(map(&[10, 11, 12])))]); + assert!(w > moved); + } + + /// advance_at refuses, and changes nothing, for a version at or before + /// the last one applied (events would leave version order) and for a + /// version the clock never issued (a fabricated version). + #[test] + fn advance_at_refuses_an_old_or_unissued_version() { + let mut t = MapTable::new(1, 1); + let early = t.claim(); + let late = t.claim(); + assert_eq!( + t.advance_at(late, 1, 1, vec![(1, Some(map(&[5])))]), + Ok(late) + ); + assert_eq!( + t.advance_at(early, 1, 1, vec![(1, Some(map(&[6])))]), + Err(AdvanceRefusal::NotAfterLast { + last: late, + version: early + }) + ); + let forged = RouteVersion(t.current_version().0 + 5); + assert_eq!( + t.advance_at(forged, 1, 1, vec![(1, Some(map(&[7])))]), + Err(AdvanceRefusal::NotIssued { + version: forged, + current: t.current_version() + }) + ); + let d = t.diagnose_at(late, 1).unwrap(); + assert_eq!((d.len(), d.global(0)), (1, Some(5))); + assert_eq!(t.seal().version(), late); + } + /// Q2: a version past the end is a typed refusal, not a clamp. #[test] fn diagnose_at_refuses_a_version_past_the_end() { From 031185076ba3215ed35130c08e2ba2a9e69686f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:28:42 +0000 Subject: [PATCH 05/14] wal-shootout: DR64 sizes measured; DR map advances use LanceCycleWriter N = 1, 4, 16, 64 tiles per advance, three processes: L commits in 2.2-5.7 ms p50 against 7.4-8.3 ms for MemWAL at a 10 ms flush, and keeps one Lance version per advance, which DR's as-of-route-version reads need. Its cost is files and scan time, which need a compaction schedule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .claude/plans/wal-shootout-v1.md | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.claude/plans/wal-shootout-v1.md b/.claude/plans/wal-shootout-v1.md index 6feef49a9..76a9dd393 100644 --- a/.claude/plans/wal-shootout-v1.md +++ b/.claude/plans/wal-shootout-v1.md @@ -252,3 +252,37 @@ compactor) does to both curves. - **D-LNC-7, decided:** `LanceCycleWriter` stays the default with the capped session; MemWAL stays an option. The 60-minute soak is deferred. + +## 8. DR64 sizes (2026-11-10) — the WAL for DR/DT64 map advances + +A DR64 tile is 64 global ids × 8 B = one 512-byte cast, so a DR advance that +changes N tiles is one cycle of N casts. `examples/wal_shootout.rs dr64` runs +N = 1, 4, 16, 64 at 200 cycles each, on one AVX-512 VM, three separate +processes. The arms are the same as §1: L = `LanceCycleWriter`, M10 / M100 = +lancedb MemWAL flushed every 10 / 100 ms. Ranges below cover the three runs. + +| N tiles | L p50 | L p99 | M10 p50 | M10 p99 | M100 p50 | +|---|---|---|---|---|---| +| 1 | 2.2–4.4 ms | 4.0–7.1 ms | 8.1–8.3 ms | 10.7–11.2 ms | 97.7–98.2 ms | +| 4 | 2.8–5.7 ms | 5.6–9.2 ms | 7.5–8.3 ms | 10.8–11.8 ms | 97.2–98.3 ms | +| 16 | 3.4–5.4 ms | 5.2–9.0 ms | 7.8–8.3 ms | 10.9–11.1 ms | 97.4–97.5 ms | +| 64 | 4.3–5.4 ms | 6.2–9.3 ms | 7.4–7.9 ms | 11.0–11.2 ms | 97.0–97.4 ms | + +| N tiles | L MiB / files / versions / scan | M10 MiB / files / versions / scan | +|---|---|---| +| 1 | 2.7 / 601 / 200 / 23–28 ms | 1.0 / 209 / 3 / 6–7 ms | +| 64 | 15.6 / 601 / 200 / 23–25 ms | 14.9 / 209 / 3 / 8–9 ms | + +- **L is faster per advance at every N measured**, at p50 and at p99, in all + three runs. M's commit time is its flush interval, not its write. +- **L keeps one Lance version per advance** (200 versions for 200 cycles). M + folds 200 advances into 3 versions. DR reads the map table as of a route + version, so a version per advance is the shape DR needs, not overhead. +- **L's cost is files and scan time:** 601 files and a 23–32 ms scan after 200 + advances, against 209 files and 5–9 ms for M. Both grow with the number of + advances, and L's needs compaction (an owner-scheduled step; not part of the + advance path). + +**D-DR64-WAL-1, decided:** DR/DT64 map advances go through `LanceCycleWriter` +(L) behind `CycleWriter`, fire and forget. MemWAL is not used for DR. Open: +the compaction schedule for the map table. From d87e448cdac840d78de3fc30ff02224a1d6d387c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:36:45 +0000 Subject: [PATCH 06/14] wip(dt64-lance): rebuild the map log on the fire-and-forget cycle writer (not yet compiled) Advances claim a route version, become casts on behalf of the owning mailbox, and are submitted to CycleWriter over LanceCycleWriter (D-DR64-WAL-1). advance_at runs only after the seal ticket is Ok; a failed seal is a gap and poisons further submissions. Restore replays sealed cycles and refuses landings on behalf of another mailbox. The synchronous Lance write loop and StaleWriter/Race/ClockMoved are removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/Cargo.toml | 22 +- crates/lance-graph-dt64-lance/src/lib.rs | 1087 +++++++++-------- .../lance-graph-dt64-lance/tests/map_log.rs | 466 +++++-- 3 files changed, 933 insertions(+), 642 deletions(-) diff --git a/crates/lance-graph-dt64-lance/Cargo.toml b/crates/lance-graph-dt64-lance/Cargo.toml index 6cb731a52..3d8caa4ea 100644 --- a/crates/lance-graph-dt64-lance/Cargo.toml +++ b/crates/lance-graph-dt64-lance/Cargo.toml @@ -4,10 +4,10 @@ # (own [workspace] root), like lance-graph-dir-sim-lance: dt64 stays free of # lance and arrow. # -# One MapTable::advance is one Lance commit. The DR route version travels in -# the rows; the Lance dataset version is the physical commit counter, used to -# refuse a stale writer, to detect a race, and to read the table as of a past -# commit (`checkout_version`). +# Advances are fire and forget: each one is a cycle submitted to the +# supervisor's CycleWriter, whose single background task owns a +# LanceCycleWriter (D-DR64-WAL-1). The advance is applied to the table only +# after its seal ticket resolves Ok; a failed seal is a gap. # # Build/verify: # cargo test --manifest-path crates/lance-graph-dt64-lance/Cargo.toml @@ -18,18 +18,18 @@ version = "0.1.0" edition = "2021" publish = false license = "Apache-2.0" -description = "DR's versioned global -> local map table stored in Lance: one advance per commit, replayed exactly at restore." +description = "DR's versioned global -> local map table made durable through the fire-and-forget cycle writer, replayed exactly at restore." [workspace] [dependencies] lance-graph-dt64 = { path = "../lance-graph-dt64" } -# The lance family moves in lockstep with the lance-graph root (13 / arrow -# 58); a local or file URI is enough here. -lance = { version = "13.*", default-features = false } -arrow-array = "58" -arrow-schema = "58" +lance-graph-contract = { path = "../lance-graph-contract" } +lance-graph-planner = { path = "../lance-graph-planner" } +lance-graph-supervisor = { path = "../lance-graph-supervisor", features = ["cycle-driver"] } +# LanceCycleWriter, the WAL (graph::cycle_sink, behind `planner`). +lance-graph = { path = "../lance-graph", default-features = false, features = ["planner"] } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } tempfile = "3" diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index 346f75fdb..38f554d6e 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -1,604 +1,679 @@ -//! DR's versioned global → local table ([`MapTable`]) stored as a separate, -//! Lance-versioned dataset (D-DT64-7 O5 / D10', D-LITHO Q5). +//! DR's versioned global → local table ([`MapTable`]) made durable through the +//! fire-and-forget cycle writer (D-DT64-7 O5 / D10', D-DR64-WAL-1). //! -//! [`MapTable`] is the in-memory DR log: append-only, every past version -//! answerable, frames replaying the layout they were sealed under. This crate -//! makes that log durable without changing it. +//! The owner of a DR map table never waits for storage: //! -//! - **One advance is one Lance commit.** [`LanceMapLog::advance`] writes the -//! advance's rows in one commit, and only then applies the same advance to -//! the table. An advance is in the dataset whole or not at all. -//! - **One row per local position** (the D10' working model). A tile map at a -//! version is `width` plus its `local → global` rows in local order. A -//! removal and the advance itself are marker rows. Ids only, never values. -//! - **Two clocks, never mixed.** The DR route version travels in the rows. -//! The Lance dataset version is the physical commit counter: it refuses a -//! stale writer, detects a race, and names a past commit to read with -//! [`LanceMapLog::restore_at`]. It is never written into, or read as, a -//! route version. -//! - **Restore replays, exactly.** [`LanceMapLog::restore`] reads every row in -//! one scan and re-applies each advance to a fresh table at the same route -//! version, so every version answers as before: an extension stays an -//! extension, a replacement a new generation, a removal an event. Rows that -//! do not form a valid log are refused; nothing is repaired. +//! ```text +//! owner: advance ── claim v ── casts ──try_submit(v, casts)──▶ returns v +//! writer: │ one background task owns the sink +//! ▼ +//! seal cycle v (LanceCycleWriter: one Lance version) +//! owner: settle ── ticket Ok ── advance_at(v) ticket Err ── v is a gap +//! ``` +//! +//! - **The WAL is `LanceCycleWriter` behind `CycleWriter`.** This crate writes +//! no dataset itself and keeps no confirmation ledger, replay queue or retry +//! loop. Measured at DR64 sizes it commits an advance in 2.2–5.7 ms p50 and +//! keeps one Lance version per advance (`.claude/plans/wal-shootout-v1.md` §8). +//! - **One advance is one cycle**, with cycle id = route version. The route +//! version is claimed from the table's clock when the advance is built +//! ([`MapTable::claim`]) and applied at that version only after the cycle's +//! seal ticket resolves `Ok` ([`MapTable::advance_at`]). A seal that fails +//! applies nothing: its version is a gap in the table and in the store. +//! - **Ownership travels in the casts.** Every cast is written on behalf of the +//! mailbox that owns the table (`SweepSlot::owner`). Restore refuses a +//! landing on behalf of any other mailbox. There is one writer by +//! construction: [`MapLog`] owns the table mutably and submits through the +//! writer handle it was given; it does not arbitrate between writers. +//! - **One 512-byte cast per 64 ids.** A DR64 tile's map is 64 global ids, +//! `8 × 64 = 512` bytes: exactly one artifact payload. A 256-wide tile is +//! four. Each advance also carries one marker cast with its route version, +//! source and relation-map revisions. +//! - **Restore replays, exactly.** [`restore`] reads every sealed cycle and +//! re-applies each advance at its own route version, so every version +//! answers as before. Rows that do not form a valid log are refused; nothing +//! is repaired. //! //! What a restore does **not** carry: frames. A [`Frame`] is a witness issued -//! by one table instance, so frames sealed before a restore are foreign to -//! the restored table (`Stale(ForeignFrame)`). Past versions stay readable -//! through [`MapTable::diagnose_at`]; executing at a past version after a -//! restore would need a frame issued at that version, which DR does not -//! offer (OPEN, DR owner). +//! by one table instance, so frames sealed before a restore are foreign to the +//! restored table (`Stale(ForeignFrame)`). Past versions stay readable through +//! [`MapTable::diagnose_at`]. Issuing frames at a past version after a restore +//! is a DR owner decision this crate does not make (OPEN). //! //! [`Frame`]: lance_graph_dt64::dr::Frame -use std::sync::Arc; +use std::collections::{BTreeMap, VecDeque}; -use arrow_array::{ - Array, RecordBatch, RecordBatchIterator, UInt16Array, UInt32Array, UInt64Array, UInt8Array, -}; -use arrow_schema::{ArrowError, DataType, Field, Schema}; -use lance::dataset::{WriteMode, WriteParams}; -use lance::Dataset; +use lance_graph::graph::cycle_sink::LanceCycleWriter; +use lance_graph_contract::collapse_gate::MailboxId; use lance_graph_dt64::dr::{ - LocalMap, MapRefusal, MapTable, RouteVersion, TileId, VersionClock, Width, + AdvanceRefusal, LocalMap, MapRefusal, MapTable, RouteVersion, TileId, VersionClock, Width, }; - -/// Row kinds. +use lance_graph_planner::persist_sink::{CycleId, SweepSlot, WalSink, WriteFailed}; +use lance_graph_supervisor::cycle_driver::SealedCycle; +use lance_graph_supervisor::cycle_writer::{CycleWriterHandle, SealError, SealTicket, SubmitError}; + +/// Bytes in one cast payload (the cycle writer's artifact width). +pub const CAST_BYTES: usize = 512; +/// Global ids carried by one cast. +const IDS_PER_CAST: usize = CAST_BYTES / 8; +/// The image row of an advance's marker cast. +const MARKER_ROW: u64 = u64::MAX; +/// Tile part that marks a removal. +const REMOVED_PART: u64 = 7; +/// An unused id slot. A map holding this id is refused before anything is +/// claimed or submitted. +pub const RESERVED_ID: u64 = u64::MAX; + +/// Marker kinds. const GENESIS: u8 = 0; const ADVANCE: u8 = 1; -const POSITION: u8 = 2; -const REMOVED: u8 = 3; - -/// One stored row. `source` / `relation_map` are set on genesis and advance -/// rows; `tile`, `width`, `local`, `global` on position rows (`tile` also on -/// removals); unused fields are zero. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct Row { - version: u64, - kind: u8, - source: u64, - relation_map: u64, - tile: u32, - width: u16, - local: u16, - global: u64, + +/// The image row of `part` of `tile`. +fn tile_row(tile: TileId, part: u64) -> u64 { + (u64::from(tile) << 3) | part } -impl Row { - fn marker(version: u64, kind: u8, source: u64, relation_map: u64) -> Row { - Row { - version, - kind, - source, - relation_map, - tile: 0, - width: 0, - local: 0, - global: 0, - } +fn parts(width: Width) -> u64 { + match width { + Width::W64 => 1, + Width::W256 => 4, } } -/// Why a save or a restore failed. +/// Why an advance was not submitted, or did not become durable. #[derive(Debug)] -pub enum StoreError { - /// Lance refused an operation. - Lance(lance::Error), - /// A stored column is missing, has another type, or holds a null. - Schema(String), - /// The stored rows do not form a valid log. - Replay(ReplayError), - /// One advance named the same tile twice. Nothing was written. +pub enum AdvanceError { + /// One advance named the same tile twice. Nothing was claimed. DuplicateTile(TileId), - /// The dataset is not at the version this writer last saw (`None`: no - /// dataset). Nothing was written. - StaleWriter { - expected: Option, - found: Option, + /// A map holds [`RESERVED_ID`]. Nothing was claimed. + ReservedId(TileId), + /// An earlier advance did not become durable, so this log submits + /// nothing further (the writer seals nothing after a failure either). + /// Nothing was claimed. + Poisoned { failed: RouteVersion }, + /// The writer refused the cycle. `version` was claimed and is a gap; the + /// casts come back inside `error`. + Submit { + version: RouteVersion, + error: SubmitError, }, - /// The commit landed, but not as the next dataset version: another - /// writer committed between this writer's check and its commit. The rows - /// are stored; the table was not advanced. - Race { expected: u64, found: u64 }, - /// The rows were stored at route version `stored`, but the table's clock - /// had moved, so the advance was issued at `issued`. Happens only when a - /// clone of the table's clock ticked between the check and the advance. - ClockMoved { stored: u64, issued: u64 }, -} - -impl From for StoreError { - fn from(e: lance::Error) -> Self { - StoreError::Lance(e) - } } -/// Why stored rows are not a valid log. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ReplayError { - /// The dataset holds no rows. - Empty, - /// The first row is not the genesis row, or a second genesis follows. - Genesis, - /// A row's route version is not after the previous advance's. - VersionOrder { previous: u64, found: u64 }, - /// A version's rows do not start with its advance marker. - MissingAdvance { version: u64 }, - /// A row kind this crate does not write. - UnknownKind(u8), - /// A width other than 64 or 256. - Width(u16), - /// A tile's positions are not `0..len` in order, or mix widths. - Positions { version: u64, tile: u32 }, - /// A tile appears twice in one advance. - DuplicateTile { version: u64, tile: u32 }, - /// The positions do not form a map (duplicate id, too many). - Map { - version: u64, - tile: u32, - refusal: MapRefusal, +/// What happened to one submitted advance once its seal resolved. +#[derive(Debug)] +pub enum Settled { + /// The cycle is durable and the advance is applied at `version`. + Applied(RouteVersion), + /// The cycle did not become durable; nothing was applied and `version` is + /// a gap. A failed seal hands its casts back inside `error`. + NotDurable { + version: RouteVersion, + error: SealError, + }, + /// The cycle is durable but the table refused to apply it. `MapLog` + /// applies its own advances in claimed order and owns its table, so its + /// API cannot produce this; it is reported rather than dropped so a + /// disagreement between store and table can never pass silently. A + /// restore replays the store. + Refused { + version: RouteVersion, + refusal: AdvanceRefusal, }, - /// Re-applying the advance issued another route version. - Reissued { stored: u64, issued: u64 }, } -fn schema() -> Arc { - Arc::new(Schema::new(vec![ - Field::new("version", DataType::UInt64, false), - Field::new("kind", DataType::UInt8, false), - Field::new("source", DataType::UInt64, false), - Field::new("relation_map", DataType::UInt64, false), - Field::new("tile", DataType::UInt32, false), - Field::new("width", DataType::UInt16, false), - Field::new("local", DataType::UInt16, false), - Field::new("global", DataType::UInt64, false), - ])) -} +/// An advance's `(source, relation_map, changes)`. +type Advance = (u64, u64, Vec<(TileId, Option)>); -fn to_batch(rows: &[Row]) -> Result { - RecordBatch::try_new( - schema(), - vec![ - Arc::new(UInt64Array::from_iter_values( - rows.iter().map(|r| r.version), - )), - Arc::new(UInt8Array::from_iter_values(rows.iter().map(|r| r.kind))), - Arc::new(UInt64Array::from_iter_values(rows.iter().map(|r| r.source))), - Arc::new(UInt64Array::from_iter_values( - rows.iter().map(|r| r.relation_map), - )), - Arc::new(UInt32Array::from_iter_values(rows.iter().map(|r| r.tile))), - Arc::new(UInt16Array::from_iter_values(rows.iter().map(|r| r.width))), - Arc::new(UInt16Array::from_iter_values(rows.iter().map(|r| r.local))), - Arc::new(UInt64Array::from_iter_values(rows.iter().map(|r| r.global))), - ], - ) +struct Pending { + version: RouteVersion, + /// `None` for the genesis cycle, which applies nothing. + advance: Option, + ticket: SealTicket, } -fn column<'a, T: 'static + Array>(b: &'a RecordBatch, name: &str) -> Result<&'a T, StoreError> { - let c = b - .column_by_name(name) - .and_then(|c| c.as_any().downcast_ref::()) - .ok_or_else(|| StoreError::Schema(format!("column `{name}`")))?; - if c.null_count() != 0 { - return Err(StoreError::Schema(format!("null in `{name}`"))); - } - Ok(c) +/// The durable owner side of one DR map table. +pub struct MapLog { + table: MapTable, + writer: CycleWriterHandle, + on_behalf: MailboxId, + next_position: u64, + pending: VecDeque, + failed: Option, } -fn from_batch(b: &RecordBatch) -> Result, StoreError> { - let version: &UInt64Array = column(b, "version")?; - let kind: &UInt8Array = column(b, "kind")?; - let source: &UInt64Array = column(b, "source")?; - let relation_map: &UInt64Array = column(b, "relation_map")?; - let tile: &UInt32Array = column(b, "tile")?; - let width: &UInt16Array = column(b, "width")?; - let local: &UInt16Array = column(b, "local")?; - let global: &UInt64Array = column(b, "global")?; - Ok((0..b.num_rows()) - .map(|i| Row { - version: version.value(i), - kind: kind.value(i), - source: source.value(i), - relation_map: relation_map.value(i), - tile: tile.value(i), - width: width.value(i), - local: local.value(i), - global: global.value(i), - }) - .collect()) -} +impl MapLog { + /// Start a new durable table on an empty store: submits the genesis + /// cycle (the table's source and relation-map revisions at its current + /// version) and returns at once. `clock` may be shared with a `RouteLog`. + pub fn create( + writer: CycleWriterHandle, + on_behalf: MailboxId, + source: u64, + relation_map: u64, + clock: VersionClock, + ) -> Result { + let table = MapTable::with_clock(source, relation_map, clock); + let version = table.current_version(); + let mut log = MapLog { + table, + writer, + on_behalf, + next_position: 0, + pending: VecDeque::new(), + failed: None, + }; + let casts = vec![log.marker(GENESIS, version, source, relation_map)]; + log.submit(version, None, casts)?; + Ok(log) + } -fn encode_width(w: Width) -> u16 { - match w { - Width::W64 => 64, - Width::W256 => 256, + /// Continue a restored table: further advances follow its last sealed + /// cycle. `writer` must be framed on the store's current head. + pub fn resume(restored: Restored, writer: CycleWriterHandle, on_behalf: MailboxId) -> MapLog { + MapLog { + table: restored.table, + writer, + on_behalf, + next_position: restored.next_position, + pending: VecDeque::new(), + failed: None, + } } -} -fn decode_width(w: u16) -> Result { - match w { - 64 => Ok(Width::W64), - 256 => Ok(Width::W256), - other => Err(ReplayError::Width(other)), + /// The table as of the last applied (durable) advance. + pub fn table(&self) -> &MapTable { + &self.table } -} -/// The rows of one advance at `version`: its marker, then each change. -fn advance_rows( - version: u64, - source: u64, - relation_map: u64, - changes: &[(TileId, Option)], -) -> Result, StoreError> { - let mut seen: Vec = changes.iter().map(|(t, _)| *t).collect(); - seen.sort_unstable(); - if let Some(w) = seen.windows(2).find(|w| w[0] == w[1]) { - return Err(StoreError::DuplicateTile(w[0])); + /// Submitted advances whose seal has not been settled yet. + pub fn pending(&self) -> usize { + self.pending.len() } - let mut rows = vec![Row::marker(version, ADVANCE, source, relation_map)]; - for (tile, map) in changes { - match map { - None => rows.push(Row { - tile: *tile, - ..Row::marker(version, REMOVED, 0, 0) - }), - Some(m) => { - let width = encode_width(m.width()); - for (local, &global) in m.view().globals().iter().enumerate() { - rows.push(Row { - tile: *tile, - width, - // A map holds at most 256 positions. - local: local as u16, - global, - ..Row::marker(version, POSITION, 0, 0) - }); - } - if m.is_empty() { - // An empty map still replaces the tile: keep it visible. - rows.push(Row { - tile: *tile, - width, - local: u16::MAX, - ..Row::marker(version, POSITION, 0, 0) - }); + + /// Submit an advance and return its claimed route version at once. It is + /// applied to [`MapLog::table`] only when [`MapLog::settle`] sees its + /// cycle sealed. + pub fn advance( + &mut self, + source: u64, + relation_map: u64, + changes: Vec<(TileId, Option)>, + ) -> Result { + if let Some(failed) = self.failed { + return Err(AdvanceError::Poisoned { failed }); + } + for (i, (tile, map)) in changes.iter().enumerate() { + if changes[..i].iter().any(|(t, _)| t == tile) { + return Err(AdvanceError::DuplicateTile(*tile)); + } + if let Some(m) = map { + if m.view().globals().contains(&RESERVED_ID) { + return Err(AdvanceError::ReservedId(*tile)); } } } + let version = self.table.claim(); + let mut casts = vec![self.marker(ADVANCE, version, source, relation_map)]; + for (tile, map) in &changes { + casts.extend(self.tile_casts(version, *tile, map.as_ref())); + } + self.submit(version, Some((source, relation_map, changes)), casts)?; + Ok(version) } - Ok(rows) -} -/// Rebuild a table from its rows, re-applying every advance at its own route -/// version. -fn replay(rows: &[Row]) -> Result { - let (genesis, rest) = rows.split_first().ok_or(ReplayError::Empty)?; - if genesis.kind != GENESIS { - return Err(ReplayError::Genesis); - } - let clock = VersionClock::new(); - while clock.current().0 < genesis.version { - clock.tick(); - } - let mut table = MapTable::with_clock(genesis.source, genesis.relation_map, clock.clone()); - let mut i = 0; - let mut previous = genesis.version; - while i < rest.len() { - let marker = rest[i]; - match marker.kind { - GENESIS => return Err(ReplayError::Genesis), - ADVANCE => {} - POSITION | REMOVED => { - return Err(ReplayError::MissingAdvance { - version: marker.version, - }) + /// Apply every advance whose seal has resolved, in submission order, + /// without waiting. Stops at the first cycle still being sealed. + pub fn settle(&mut self) -> Vec { + let mut out = Vec::new(); + while let Some(front) = self.pending.front_mut() { + let Some(result) = front.ticket.try_sealed() else { + break; + }; + let Pending { + version, advance, .. + } = self.pending.pop_front().expect("front exists"); + if let Some(s) = self.resolve(version, advance, result) { + out.push(s); } - other => return Err(ReplayError::UnknownKind(other)), } - let version = marker.version; - if version <= previous { - return Err(ReplayError::VersionOrder { - previous, - found: version, - }); + out + } + + /// Wait for the oldest submitted advance and apply it. `None` when + /// nothing is pending. Used at shutdown and in tests; the owner's loop + /// calls [`MapLog::settle`]. + pub async fn settle_next(&mut self) -> Option { + loop { + let Pending { + version, + advance, + ticket, + } = self.pending.pop_front()?; + let result = ticket.sealed().await; + if let Some(s) = self.resolve(version, advance, result) { + return Some(s); + } } - let end = i - + 1 - + rest[i + 1..] - .iter() - .take_while(|r| r.version == version && r.kind != ADVANCE) - .count(); - let changes = changes_of(version, &rest[i + 1..end])?; - while clock.current().0 + 1 < version { - clock.tick(); + } + + fn resolve( + &mut self, + version: RouteVersion, + advance: Option, + result: Result, + ) -> Option { + match result { + Err(error) => { + self.failed.get_or_insert(version); + Some(Settled::NotDurable { version, error }) + } + Ok(_) => { + let (source, relation_map, changes) = advance?; + Some( + match self + .table + .advance_at(version, source, relation_map, changes) + { + Ok(v) => Settled::Applied(v), + Err(refusal) => Settled::Refused { version, refusal }, + }, + ) + } } - let issued = table.advance(marker.source, marker.relation_map, changes); - if issued.0 != version { - return Err(ReplayError::Reissued { - stored: version, - issued: issued.0, - }); + } + + fn submit( + &mut self, + version: RouteVersion, + advance: Option, + casts: Vec, + ) -> Result<(), AdvanceError> { + match self.writer.try_submit(CycleId(version.0), casts) { + Ok(ticket) => { + self.pending.push_back(Pending { + version, + advance, + ticket, + }); + Ok(()) + } + Err(error) => Err(AdvanceError::Submit { version, error }), } - previous = version; - i = end; } - Ok(table) -} -/// The changes of one advance from its position and removal rows. -fn changes_of(version: u64, rows: &[Row]) -> Result)>, ReplayError> { - let mut changes: Vec<(TileId, Option)> = Vec::new(); - let mut j = 0; - while j < rows.len() { - let tile = rows[j].tile; - if changes.iter().any(|(t, _)| *t == tile) { - return Err(ReplayError::DuplicateTile { version, tile }); + fn cast(&mut self, version: RouteVersion, row: u64, payload: Vec) -> SweepSlot { + let stream_position = self.next_position; + self.next_position += 1; + SweepSlot { + cycle: CycleId(version.0), + stream_position, + owner: self.on_behalf, + row, + paired_move: None, + payload, } - match rows[j].kind { - REMOVED => { - changes.push((tile, None)); - j += 1; - } - POSITION => { - let run = rows[j..] - .iter() - .take_while(|r| r.kind == POSITION && r.tile == tile) - .count(); - let span = &rows[j..j + run]; - let width = decode_width(span[0].width)?; - let empty = run == 1 && span[0].local == u16::MAX; - let ordered = span - .iter() - .enumerate() - .all(|(k, r)| usize::from(r.local) == k && r.width == span[0].width); - if !empty && !ordered { - return Err(ReplayError::Positions { version, tile }); + } + + fn marker(&mut self, kind: u8, version: RouteVersion, source: u64, rm: u64) -> SweepSlot { + let mut p = vec![0u8; CAST_BYTES]; + p[0] = kind; + p[8..16].copy_from_slice(&version.0.to_le_bytes()); + p[16..24].copy_from_slice(&source.to_le_bytes()); + p[24..32].copy_from_slice(&rm.to_le_bytes()); + self.cast(version, MARKER_ROW, p) + } + + fn tile_casts( + &mut self, + version: RouteVersion, + tile: TileId, + map: Option<&LocalMap>, + ) -> Vec { + let Some(map) = map else { + let p = vec![0u8; CAST_BYTES]; + return vec![self.cast(version, tile_row(tile, REMOVED_PART), p)]; + }; + let view = map.view(); + let globals = view.globals(); + (0..parts(map.width())) + .map(|part| { + let mut p = vec![0u8; CAST_BYTES]; + for slot in 0..IDS_PER_CAST { + let i = part as usize * IDS_PER_CAST + slot; + let id = globals.get(i).copied().unwrap_or(RESERVED_ID); + p[slot * 8..slot * 8 + 8].copy_from_slice(&id.to_le_bytes()); } - let globals: Vec = if empty { - Vec::new() - } else { - span.iter().map(|r| r.global).collect() - }; - let map = - LocalMap::prepare(width, &globals).map_err(|refusal| ReplayError::Map { - version, - tile, - refusal, - })?; - changes.push((tile, Some(map))); - j += run; - } - other => return Err(ReplayError::UnknownKind(other)), - } + self.cast(version, tile_row(tile, part), p) + }) + .collect() } - Ok(changes) } -fn is_not_found(e: &lance::Error) -> bool { - matches!(e, lance::Error::DatasetNotFound { .. }) +/// A sealed store that can hand back each cycle's coalesced image (the +/// payloads `WalSink::scan_sealed` deliberately does not carry). +#[allow(async_fn_in_trait)] +pub trait SealedImages: WalSink { + /// `row → payload` for one sealed cycle. + async fn sealed_image(&self, cycle: CycleId) -> Result>, WriteFailed>; } -/// The dataset version a commit on top of `seen` makes: Lance numbers a new -/// dataset 1 and each later commit one higher. -fn next_version(seen: Option) -> u64 { - seen.map_or(1, |v| v + 1) +impl SealedImages for LanceCycleWriter { + async fn sealed_image(&self, cycle: CycleId) -> Result>, WriteFailed> { + self.scan_image(cycle).await + } } -/// A writer's handle on one DR map table's dataset at `uri`. -#[derive(Debug, Clone)] -pub struct LanceMapLog { - uri: String, - /// The dataset version this writer last saw. - version: Option, +/// A table rebuilt from its store. +pub struct Restored { + pub table: MapTable, + /// The next cast stream position (one past the last stored landing). + pub next_position: u64, + /// The last sealed cycle (the last advance's route version). + pub last: RouteVersion, } -impl LanceMapLog { - /// Create the dataset with its genesis row and return a new table with - /// the given revisions and its writer. Refuses if a dataset is already at - /// `uri`. - pub async fn create( - uri: impl Into, - source: u64, - relation_map: u64, - ) -> Result<(MapTable, LanceMapLog), StoreError> { - let mut log = LanceMapLog { - uri: uri.into(), - version: None, - }; - let table = MapTable::new(source, relation_map); - let genesis = Row::marker(table.current_version().0, GENESIS, source, relation_map); - log.commit(&[genesis]).await?; - Ok((table, log)) - } +/// Why a restore failed. +#[derive(Debug)] +pub enum RestoreError { + /// The store could not be read. + Read(WriteFailed), + /// The stored cycles do not form a valid log. + Replay(ReplayError), +} - /// The dataset version this writer last saw (the physical commit - /// counter, not a route version). - pub fn dataset_version(&self) -> Option { - self.version - } +/// Why stored cycles are not a valid log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReplayError { + /// The store holds no cycles. + Empty, + /// The first cycle is not a genesis, or a second genesis follows. + Genesis { cycle: u64 }, + /// A cycle has no marker cast. + MissingMarker { cycle: u64 }, + /// A marker's route version is not its cycle id. + VersionMismatch { cycle: u64, marker: u64 }, + /// A marker kind this crate does not write. + UnknownKind { cycle: u64, kind: u8 }, + /// A landing was written on behalf of another mailbox. + ForeignOwner { cycle: u64, owner: MailboxId }, + /// A cast length other than [`CAST_BYTES`]. + PayloadLength { cycle: u64, row: u64, len: usize }, + /// A tile's parts are not one width's parts or one removal, or its ids + /// are not a prefix followed only by unused slots. + Positions { cycle: u64, tile: TileId }, + /// The ids do not form a map (duplicate id, too many). + Map { + cycle: u64, + tile: TileId, + refusal: MapRefusal, + }, + /// Re-applying the advance was refused by the table. + Apply { cycle: u64, refusal: AdvanceRefusal }, +} - /// Store one advance as one Lance commit, then apply it to `table`. - /// Returns the route version it was issued at. - /// - /// # Errors - /// - /// - [`StoreError::DuplicateTile`] or [`StoreError::StaleWriter`]: - /// nothing is written and the table is unchanged. - /// - [`StoreError::Race`]: the rows landed as another dataset version; - /// the table is unchanged. - /// - [`StoreError::ClockMoved`]: see the variant. - pub async fn advance( - &mut self, - table: &mut MapTable, - source: u64, - relation_map: u64, - changes: Vec<(TileId, Option)>, - ) -> Result { - let stored = table.current_version().0 + 1; - let rows = advance_rows(stored, source, relation_map, &changes)?; - self.commit(&rows).await?; - let issued = table.advance(source, relation_map, changes); - if issued.0 != stored { - return Err(StoreError::ClockMoved { - stored, - issued: issued.0, - }); +/// Rebuild a table from every sealed cycle in `sink`, re-applying each +/// advance at its own route version. Every landing must be on behalf of +/// `owner`. `clock` becomes the restored table's clock. +pub async fn restore( + sink: &S, + owner: MailboxId, + clock: VersionClock, +) -> Result { + let landings = sink.scan_sealed(None).await.map_err(RestoreError::Read)?; + let mut cycles: Vec = Vec::new(); + let mut next_position = 0; + for l in &landings { + if l.slot.owner != owner { + return Err(RestoreError::Replay(ReplayError::ForeignOwner { + cycle: l.cycle.0, + owner: l.slot.owner, + })); + } + next_position = next_position.max(l.slot.stream_position + 1); + if cycles.last() != Some(&l.cycle.0) { + cycles.push(l.cycle.0); } - Ok(issued) } + let mut images = Vec::with_capacity(cycles.len()); + for &c in &cycles { + let image = sink + .sealed_image(CycleId(c)) + .await + .map_err(RestoreError::Read)?; + images.push((c, image)); + } + let (table, last) = replay(&images, clock).map_err(RestoreError::Replay)?; + Ok(Restored { + table, + next_position, + last, + }) +} - async fn commit(&mut self, rows: &[Row]) -> Result { - let found = match Dataset::open(&self.uri).await { - Ok(ds) => Some(ds.latest_version_id().await?), - Err(e) if is_not_found(&e) => None, - Err(e) => return Err(e.into()), - }; - if found != self.version { - return Err(StoreError::StaleWriter { - expected: self.version, - found, - }); +/// A decoded marker: `(kind, version, source, relation_map)`. +fn decode_marker(cycle: u64, image: &BTreeMap>) -> Result<(u8, u64, u64, u64), ReplayError> { + let p = image + .get(&MARKER_ROW) + .ok_or(ReplayError::MissingMarker { cycle })?; + let word = |at: usize| u64::from_le_bytes(p[at..at + 8].try_into().expect("8 bytes")); + Ok((p[0], word(8), word(16), word(24))) +} + +fn replay( + images: &[(u64, BTreeMap>)], + clock: VersionClock, +) -> Result<(MapTable, RouteVersion), ReplayError> { + for (cycle, image) in images { + for (row, p) in image { + if p.len() != CAST_BYTES { + return Err(ReplayError::PayloadLength { + cycle: *cycle, + row: *row, + len: p.len(), + }); + } } - let rb = to_batch(rows).map_err(|e| StoreError::Schema(e.to_string()))?; - let reader = RecordBatchIterator::new([Ok(rb)], schema()); - let mode = if found.is_some() { - WriteMode::Append - } else { - WriteMode::Create - }; - let params = WriteParams { - mode, - ..WriteParams::default() - }; - let ds = match Dataset::write(reader, self.uri.as_str(), Some(params)).await { - Ok(ds) => ds, - Err(lance::Error::DatasetAlreadyExists { .. }) => { - return Err(StoreError::StaleWriter { - expected: None, - found: Dataset::open(&self.uri) - .await - .ok() - .map(|d| d.version().version), + } + let ((first, genesis), rest) = images.split_first().ok_or(ReplayError::Empty)?; + let (kind, version, source, relation_map) = decode_marker(*first, genesis)?; + if kind != GENESIS || genesis.len() != 1 { + return Err(ReplayError::Genesis { cycle: *first }); + } + if version != *first { + return Err(ReplayError::VersionMismatch { + cycle: *first, + marker: version, + }); + } + while clock.current().0 < version { + clock.tick(); + } + let mut table = MapTable::with_clock(source, relation_map, clock.clone()); + let mut last = RouteVersion(version); + for (cycle, image) in rest { + let (kind, version, source, relation_map) = decode_marker(*cycle, image)?; + match kind { + ADVANCE => {} + GENESIS => return Err(ReplayError::Genesis { cycle: *cycle }), + other => { + return Err(ReplayError::UnknownKind { + cycle: *cycle, + kind: other, }) } - Err(e) => return Err(e.into()), - }; - let made = ds.version().version; - let next = next_version(found); - if made != next { - return Err(StoreError::Race { - expected: next, - found: made, + } + if version != *cycle { + return Err(ReplayError::VersionMismatch { + cycle: *cycle, + marker: version, }); } - self.version = Some(made); - Ok(made) + let changes = changes_of(*cycle, image)?; + while clock.current().0 < version { + clock.tick(); + } + table + .advance_at(RouteVersion(version), source, relation_map, changes) + .map_err(|refusal| ReplayError::Apply { + cycle: *cycle, + refusal, + })?; + last = RouteVersion(version); } + Ok((table, last)) +} - /// Read the whole table at `uri` once and replay it, with a writer at the - /// dataset's current version. - pub async fn restore(uri: impl Into) -> Result<(MapTable, LanceMapLog), StoreError> { - let uri = uri.into(); - let ds = Dataset::open(&uri).await?; - let table = read(&ds).await?; - let version = Some(ds.version().version); - Ok((table, LanceMapLog { uri, version })) +/// The changes of one advance from its tile casts. +fn changes_of( + cycle: u64, + image: &BTreeMap>, +) -> Result)>, ReplayError> { + let mut by_tile: BTreeMap> = BTreeMap::new(); + for (&row, p) in image { + if row == MARKER_ROW { + continue; + } + let tile = TileId::try_from(row >> 3).map_err(|_| ReplayError::Positions { + cycle, + tile: TileId::MAX, + })?; + by_tile.entry(tile).or_default().push((row & 7, p)); } - - /// The table as of dataset version `dataset_version` (a past commit), - /// read through Lance's own version history. Read-only: no writer is - /// returned, since appending on top of a past commit would fork the log. - pub async fn restore_at( - uri: impl Into, - dataset_version: u64, - ) -> Result { - let uri = uri.into(); - let ds = Dataset::open(&uri) - .await? - .checkout_version(dataset_version) - .await?; - read(&ds).await + let mut changes = Vec::with_capacity(by_tile.len()); + for (tile, mut rows) in by_tile { + rows.sort_by_key(|(part, _)| *part); + let found: Vec = rows.iter().map(|(part, _)| *part).collect(); + let bad = ReplayError::Positions { cycle, tile }; + let width = match found.as_slice() { + [REMOVED_PART] => { + changes.push((tile, None)); + continue; + } + [0] => Width::W64, + [0, 1, 2, 3] => Width::W256, + _ => return Err(bad), + }; + let mut globals = Vec::new(); + let mut ended = false; + for (_, p) in &rows { + for slot in p.chunks_exact(8) { + let id = u64::from_le_bytes(slot.try_into().expect("8 bytes")); + if id == RESERVED_ID { + ended = true; + } else if ended { + return Err(bad); + } else { + globals.push(id); + } + } + } + let map = LocalMap::prepare(width, &globals).map_err(|refusal| ReplayError::Map { + cycle, + tile, + refusal, + })?; + changes.push((tile, Some(map))); } -} - -async fn read(ds: &Dataset) -> Result { - let all = ds.scan().try_into_batch().await?; - let rows = from_batch(&all)?; - replay(&rows).map_err(StoreError::Replay) + Ok(changes) } #[cfg(test)] mod tests { use super::*; - fn map(w: Width, ids: &[u64]) -> Option { - Some(LocalMap::prepare(w, ids).unwrap()) + fn image(entries: &[(u64, Vec)]) -> BTreeMap> { + entries.iter().cloned().collect() } - #[test] - fn rows_round_trip_through_a_record_batch() { - let rows = advance_rows(7, 3, 4, &[(1, map(Width::W64, &[9, 8])), (2, None)]).unwrap(); - assert_eq!(rows.len(), 4); - assert_eq!(from_batch(&to_batch(&rows).unwrap()).unwrap(), rows); + fn ids(list: &[u64]) -> Vec { + let mut p = vec![0u8; CAST_BYTES]; + for slot in 0..IDS_PER_CAST { + let id = list.get(slot).copied().unwrap_or(RESERVED_ID); + p[slot * 8..slot * 8 + 8].copy_from_slice(&id.to_le_bytes()); + } + p } #[test] - fn an_advance_naming_a_tile_twice_is_refused() { - let err = advance_rows(1, 1, 1, &[(5, None), (5, map(Width::W64, &[1]))]).unwrap_err(); - assert!(matches!(err, StoreError::DuplicateTile(5))); + fn a_w64_tile_is_one_cast_and_reads_back_as_its_ids() { + let c = changes_of(1, &image(&[(tile_row(5, 0), ids(&[9, 8, 7]))])).unwrap(); + assert_eq!(c.len(), 1); + let (tile, map) = &c[0]; + assert_eq!(*tile, 5); + let map = map.as_ref().unwrap(); + assert_eq!(map.width(), Width::W64); + assert_eq!(map.view().globals(), &[9, 8, 7]); } #[test] - fn replay_refuses_rows_that_are_not_a_log() { - let genesis = Row::marker(0, GENESIS, 1, 1); - let good = advance_rows(1, 1, 1, &[(1, map(Width::W64, &[4, 5]))]).unwrap(); - let mut rows = vec![genesis]; - rows.extend(&good); - assert!(replay(&rows).is_ok()); - - assert_eq!(replay(&rows[1..]).unwrap_err(), ReplayError::Genesis); - assert_eq!(replay(&[]).unwrap_err(), ReplayError::Empty); - - let mut swapped = rows.clone(); - swapped.swap(2, 3); + fn tile_parts_that_are_not_one_width_are_refused() { + // Two of a W256 tile's four parts. + let two = image(&[(tile_row(1, 0), ids(&[1])), (tile_row(1, 1), ids(&[]))]); assert_eq!( - replay(&swapped).unwrap_err(), - ReplayError::Positions { - version: 1, - tile: 1 - } + changes_of(3, &two).unwrap_err(), + ReplayError::Positions { cycle: 3, tile: 1 } ); + // A removal next to a part. + let mixed = image(&[ + (tile_row(1, 0), ids(&[1])), + (tile_row(1, REMOVED_PART), vec![0; CAST_BYTES]), + ]); + assert!(changes_of(3, &mixed).is_err()); + } - let mut dup = rows.clone(); - dup[3].global = dup[2].global; - assert!(matches!( - replay(&dup).unwrap_err(), - ReplayError::Map { - refusal: MapRefusal::Duplicate, - .. - } - )); + #[test] + fn an_id_after_an_unused_slot_is_refused() { + let mut p = ids(&[1, 2]); + p[5 * 8..6 * 8].copy_from_slice(&3u64.to_le_bytes()); + let err = changes_of(4, &image(&[(tile_row(2, 0), p)])).unwrap_err(); + assert_eq!(err, ReplayError::Positions { cycle: 4, tile: 2 }); + } - let mut back = rows.clone(); - back.extend(advance_rows(1, 1, 1, &[]).unwrap()); + #[test] + fn replay_refuses_a_log_without_genesis_or_marker() { + let advance = |v: u64| { + let mut p = vec![0u8; CAST_BYTES]; + p[0] = ADVANCE; + p[8..16].copy_from_slice(&v.to_le_bytes()); + p + }; + let genesis = { + let mut p = vec![0u8; CAST_BYTES]; + p[0] = GENESIS; + p + }; + let replay_of = |cycles: Vec<(u64, BTreeMap>)>| { + replay(&cycles, VersionClock::new()).map(|_| ()).unwrap_err() + }; + assert_eq!(replay_of(vec![]), ReplayError::Empty); assert_eq!( - replay(&back).unwrap_err(), - ReplayError::VersionOrder { - previous: 1, - found: 1 + replay_of(vec![(1, image(&[(MARKER_ROW, advance(1))]))]), + ReplayError::Genesis { cycle: 1 } + ); + assert_eq!( + replay_of(vec![ + (0, image(&[(MARKER_ROW, genesis.clone())])), + (1, image(&[(tile_row(1, 0), ids(&[1]))])), + ]), + ReplayError::MissingMarker { cycle: 1 } + ); + assert_eq!( + replay_of(vec![ + (0, image(&[(MARKER_ROW, genesis.clone())])), + (2, image(&[(MARKER_ROW, advance(1))])), + ]), + ReplayError::VersionMismatch { + cycle: 2, + marker: 1 } ); - - let orphan = vec![genesis, good[1]]; assert_eq!( - replay(&orphan).unwrap_err(), - ReplayError::MissingAdvance { version: 1 } + replay_of(vec![ + (0, image(&[(MARKER_ROW, genesis.clone())])), + (1, image(&[(MARKER_ROW, genesis)])), + ]), + ReplayError::Genesis { cycle: 1 } ); } - - #[test] - fn the_next_version_is_one_more_or_one_for_a_new_dataset() { - assert_eq!(next_version(None), 1); - assert_eq!(next_version(Some(4)), 5); - } } diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs index fbe3b0316..d83bef33a 100644 --- a/crates/lance-graph-dt64-lance/tests/map_log.rs +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -1,11 +1,34 @@ -//! A DR map table survives a round trip through its Lance dataset exactly: -//! every past route version answers as before, one advance is one commit, -//! a past commit reads the table as of then, a stale writer is refused, and -//! route versions are never confused with dataset versions. +//! A DR map table made durable through the fire-and-forget cycle writer. +//! +//! - The owner submits and returns at once; an advance is applied only after +//! its cycle seals, at the route version claimed when it was built. +//! - A failed seal applies nothing and leaves a gap. +//! - Every landing is on behalf of the owning mailbox. +//! - A restore answers every past version exactly. +//! +//! The interleavings are driven deterministically by a gated in-memory sink +//! (`GateWal`): commits wait on a semaphore the test opens, and a chosen cycle +//! fails. The round trip runs on the real `LanceCycleWriter`. -use lance_graph_dt64::dr::{FrameRefusal, LocalMap, MapRead, MapTable, RouteVersion, Width}; -use lance_graph_dt64_lance::{LanceMapLog, StoreError}; +use std::collections::BTreeMap; +use std::sync::Arc; +use lance_graph::graph::cycle_sink::LanceCycleWriter; +use lance_graph_contract::scheduler::DatasetVersion; +use lance_graph_dt64::dr::{ + FrameRefusal, LocalMap, MapRead, MapTable, RouteVersion, VersionClock, Width, +}; +use lance_graph_dt64_lance::{ + restore, AdvanceError, MapLog, ReplayError, RestoreError, SealedImages, Settled, RESERVED_ID, +}; +use lance_graph_planner::persist_sink::{ + CommitError, CommitOutcome, CycleId, DetachedCycleBatch, FrameMeta, LandedSlot, WalSink, + WriteFailed, +}; +use lance_graph_supervisor::cycle_writer::{cycle_writer, SealError}; +use tokio::sync::Semaphore; + +const OWNER: u32 = 42; const TILES: [u32; 3] = [1, 2, 3]; fn map(w: Width, ids: &[u64]) -> Option { @@ -40,8 +63,8 @@ fn assert_same(a: &MapTable, b: &MapTable, upto: u64) { } /// Six advances covering every kind of history event. -async fn history(log: &mut LanceMapLog, t: &mut MapTable) { - let steps: Vec)>> = vec![ +fn history() -> Vec)>> { + vec![ vec![ (1, map(Width::W64, &[10, 11, 12])), (2, map(Width::W256, &[7, 3, 9, 1])), @@ -51,146 +74,339 @@ async fn history(log: &mut LanceMapLog, t: &mut MapTable) { vec![(2, None)], // removal vec![], // advance with no change vec![(2, map(Width::W64, &[7])), (3, map(Width::W256, &[]))], // re-add, empty map - ]; - for (k, changes) in steps.into_iter().enumerate() { - let v = log.advance(t, 1, 1 + k as u64, changes).await.unwrap(); - assert_eq!(v, RouteVersion(k as u64 + 1)); + ] +} + +/// One-writer in-memory sink. Fences on `base_version`, publishes head + 1, +/// keeps each cycle's landings and image, can hold commits behind a gate and +/// can fail a chosen cycle. Like the real writer, scanned landings carry no +/// payload. +#[derive(Default)] +struct GateWal { + head: u64, + cycles: Vec<(CycleId, Vec, BTreeMap>)>, + gate: Option>, + fail_on: Option, +} + +impl WalSink for GateWal { + async fn commit_cycle( + &mut self, + batch: DetachedCycleBatch, + ) -> Result { + if let Some(gate) = &self.gate { + gate.acquire().await.expect("gate open").forget(); + } + if Some(batch.frame.cycle) == self.fail_on { + return Err(CommitError::Io(WriteFailed("injected".into()))); + } + if batch.frame.base_version.0 != self.head { + return Err(CommitError::Fenced { + current_head: DatasetVersion(self.head), + }); + } + self.head += 1; + let landings = batch + .landings + .iter() + .map(|s| { + let mut slot = s.clone(); + slot.payload = Vec::new(); + LandedSlot { + cycle: batch.frame.cycle, + slot, + } + }) + .collect(); + self.cycles + .push((batch.frame.cycle, landings, batch.image.clone())); + Ok(CommitOutcome::Committed { + version: DatasetVersion(self.head), + cycle: batch.frame.cycle, + batch_hash: batch.batch_hash, + }) + } + + async fn scan_sealed(&self, after: Option) -> Result, WriteFailed> { + Ok(self + .cycles + .iter() + .filter(|(c, _, _)| after.is_none_or(|a| *c > a)) + .flat_map(|(_, l, _)| l.iter().cloned()) + .collect()) + } + + async fn timeline(&self) -> Result, WriteFailed> { + Ok(Vec::new()) + } +} + +impl SealedImages for GateWal { + async fn sealed_image(&self, cycle: CycleId) -> Result>, WriteFailed> { + Ok(self + .cycles + .iter() + .find(|(c, _, _)| *c == cycle) + .map(|(_, _, i)| i.clone()) + .unwrap_or_default()) + } +} + +fn applied(s: Option) -> RouteVersion { + match s { + Some(Settled::Applied(v)) => v, + other => panic!("expected Applied, got {other:?}"), } } #[tokio::test(flavor = "multi_thread")] async fn a_restored_table_answers_every_past_version_exactly() { let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); - history(&mut log, &mut t).await; - - let (r, rlog) = LanceMapLog::restore(&uri).await.unwrap(); - assert_eq!(r.current_version(), t.current_version()); - assert_eq!(r.revisions(), t.revisions()); - assert_same(&t, &r, t.current_version().0); + let path = dir.path().join("map.lance"); + let sink = LanceCycleWriter::open(path.to_str().unwrap()).await.unwrap(); + let head = sink.head(); + let (writer, handle) = cycle_writer(sink, head, 8); + + let (exit, live) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + for (k, changes) in history().into_iter().enumerate() { + let v = log.advance(1, 2 + k as u64, changes).unwrap(); + assert_eq!(v, RouteVersion(k as u64 + 1)); + } + assert!(log.settle_next().await.is_none(), "genesis applies nothing"); + for k in 1..=6 { + assert_eq!(applied(log.settle_next().await), RouteVersion(k)); + } + log + }); + let live = live.table(); + + // One Lance version per cycle: genesis plus six advances. + assert_eq!(exit.sink.head(), DatasetVersion(head.0 + 7)); + + let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + assert_eq!(restored.last, RouteVersion(6)); + assert_same(live, &restored.table, 6); for tile in TILES { assert_eq!( - r.generation_count(tile), - t.generation_count(tile), - "tile {tile}" + live.generation_count(tile), + restored.table.generation_count(tile) ); - assert_eq!(r.event_count(tile), t.event_count(tile), "tile {tile}"); + assert_eq!(live.event_count(tile), restored.table.event_count(tile)); } - // The extension is stored as an extension: tile 1 has two generations - // (the original, extended in place, and the reorder), not three. - assert_eq!(r.generation_count(1), 2); - // Genesis plus six advances: seven commits. - assert_eq!(log.dataset_version(), Some(7)); - assert_eq!(rlog.dataset_version(), Some(7)); } -/// Lance's own version history is the DR history: dataset version `k` holds -/// genesis and the first `k - 1` advances, and answers each of them as the -/// live table does. -#[tokio::test(flavor = "multi_thread")] -async fn each_past_commit_reads_the_table_as_of_then() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); - history(&mut log, &mut t).await; - for k in 1..=7u64 { - let past = LanceMapLog::restore_at(&uri, k).await.unwrap(); +#[tokio::test] +async fn the_owner_returns_before_the_seal_and_applies_only_after_it() { + let gate = Arc::new(Semaphore::new(0)); + let sink = GateWal { + gate: Some(gate.clone()), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + + let (_exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let v1 = log.advance(1, 1, vec![(1, map(Width::W64, &[5]))]).unwrap(); + let v2 = log.advance(1, 1, vec![(1, map(Width::W64, &[5, 6]))]).unwrap(); + assert_eq!((v1, v2), (RouteVersion(1), RouteVersion(2))); + + // Nothing sealed yet: the table has not moved. + for _ in 0..100 { + tokio::task::yield_now().await; + } + assert!(log.settle().is_empty()); + assert_eq!(log.pending(), 3); assert_eq!( - past.current_version(), - RouteVersion(k - 1), - "dataset version {k}" + answer(log.table(), 2, 1), + Err(MapRead::NoSuchTile(1)), + "unsealed advances must not be applied" ); - assert_same(&t, &past, k - 1); - } + + gate.add_permits(3); + assert!(log.settle_next().await.is_none()); + assert_eq!(applied(log.settle_next().await), v1); + assert_eq!(applied(log.settle_next().await), v2); + assert_eq!(answer(log.table(), 2, 1), Ok((64, vec![5, 6]))); + assert_eq!(answer(log.table(), 1, 1), Ok((64, vec![5]))); + }); } -#[tokio::test(flavor = "multi_thread")] -async fn a_stale_writer_writes_nothing() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); - let (mut t2, mut log2) = LanceMapLog::restore(&uri).await.unwrap(); - log.advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[1, 2]))]) - .await - .unwrap(); - - let before = t2.current_version(); - let err = log2 - .advance(&mut t2, 1, 1, vec![(1, map(Width::W64, &[5]))]) - .await - .unwrap_err(); - assert!(matches!( - err, - StoreError::StaleWriter { - expected: Some(1), - found: Some(2) +#[tokio::test] +async fn a_clock_tick_between_claim_and_seal_lands_at_the_claimed_version() { + let gate = Arc::new(Semaphore::new(0)); + let sink = GateWal { + gate: Some(gate.clone()), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + + let (exit, ()) = tokio::join!(writer.run(), async move { + let clock = VersionClock::new(); + let mut log = MapLog::create(handle, OWNER, 1, 1, clock.clone()).unwrap(); + let v = log.advance(1, 1, vec![(1, map(Width::W64, &[9]))]).unwrap(); + // Another holder of the clock (a RouteLog) ticks while the cycle seals. + clock.tick(); + clock.tick(); + gate.add_permits(2); + assert!(log.settle_next().await.is_none()); + assert_eq!(applied(log.settle_next().await), v); + assert_eq!(log.table().current_version(), RouteVersion(v.0 + 2)); + assert_eq!(answer(log.table(), v.0, 1), Ok((64, vec![9]))); + assert_eq!(answer(log.table(), v.0 - 1, 1), Err(MapRead::NoSuchTile(1))); + }); + + let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + assert_eq!(restored.last, RouteVersion(1)); + assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![9]))); +} + +#[tokio::test] +async fn a_failed_seal_applies_nothing_and_poisons_what_follows() { + let gate = Arc::new(Semaphore::new(0)); + let sink = GateWal { + gate: Some(gate.clone()), + fail_on: Some(CycleId(2)), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + + let (exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let v1 = log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); + let v2 = log.advance(1, 1, vec![(1, map(Width::W64, &[1, 2]))]).unwrap(); + let v3 = log.advance(1, 1, vec![(2, map(Width::W64, &[3]))]).unwrap(); + gate.add_permits(4); + + assert!(log.settle_next().await.is_none()); + assert_eq!(applied(log.settle_next().await), v1); + match log.settle_next().await { + Some(Settled::NotDurable { + version, + error: SealError::Failed(f), + }) => { + assert_eq!(version, v2); + assert_eq!(f.casts.len(), 2, "marker + one tile cast come back"); + } + other => panic!("expected the failed seal, got {other:?}"), } - )); - assert_eq!(t2.current_version(), before, "the table was not advanced"); - let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); - assert_eq!( - answer(&r, 1, 1), - Ok((64, vec![1, 2])), - "only the first writer's advance is stored" - ); -} - -/// A table whose clock is shared (here: ticked from a clone, as a RouteLog -/// sharing it would) issues route versions with gaps. They are stored and -/// replayed as route versions, independent of the dataset version. -#[tokio::test(flavor = "multi_thread")] -async fn route_versions_with_gaps_replay_as_route_versions() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); - let shared = t.clock(); - for _ in 0..5 { - shared.tick(); + match log.settle_next().await { + Some(Settled::NotDurable { + version, + error: SealError::Poisoned { casts, .. }, + }) => { + assert_eq!(version, v3); + assert_eq!(casts.len(), 2); + } + other => panic!("expected the poisoned seal, got {other:?}"), + } + + // Neither v2 nor v3 was applied. + assert_eq!(answer(log.table(), v2.0, 1), Ok((64, vec![1]))); + assert_eq!(answer(log.table(), v3.0, 2), Err(MapRead::NoSuchTile(2))); + // And nothing more is submitted. + let before = log.table().current_version(); + match log.advance(1, 1, vec![]) { + Err(AdvanceError::Poisoned { failed }) => assert_eq!(failed, v2), + other => panic!("expected Poisoned, got {other:?}"), + } + assert_eq!(log.table().current_version(), before, "nothing was claimed"); + }); + + // The store holds genesis and v1 only, and replays to the same table. + let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + assert_eq!(restored.last, RouteVersion(1)); + assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![1]))); + assert_eq!(answer(&restored.table, 1, 2), Err(MapRead::NoSuchTile(2))); +} + +#[tokio::test] +async fn every_landing_is_on_behalf_of_the_owner() { + let (writer, handle) = cycle_writer(GateWal::default(), DatasetVersion(0), 8); + let (exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + log.advance(1, 1, vec![(1, map(Width::W256, &[1, 2]))]) + .unwrap(); + while log.settle_next().await.is_some() {} + }); + let landings = exit.sink.scan_sealed(None).await.unwrap(); + // genesis marker + advance marker + four W256 parts + assert_eq!(landings.len(), 6); + assert!(landings.iter().all(|l| l.slot.owner == OWNER)); + assert!(landings.iter().all(|l| l.slot.paired_move.is_none())); + + match restore(&exit.sink, OWNER + 1, VersionClock::new()).await { + Err(RestoreError::Replay(ReplayError::ForeignOwner { owner, .. })) => { + assert_eq!(owner, OWNER) + } + Ok(_) => panic!("a restore on behalf of another mailbox must be refused"), + Err(e) => panic!("expected ForeignOwner, got {e:?}"), } - let v = log - .advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[42]))]) - .await - .unwrap(); - assert_eq!(v, RouteVersion(6)); - assert_eq!(log.dataset_version(), Some(2), "two commits, not six"); +} - let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); - assert_eq!(r.current_version(), RouteVersion(6)); - assert_eq!(answer(&r, 5, 1), Err(MapRead::NoSuchTile(1))); - assert_eq!(answer(&r, 6, 1), Ok((64, vec![42]))); +#[tokio::test] +async fn a_reserved_id_or_a_repeated_tile_is_refused_before_a_version_is_claimed() { + let (writer, handle) = cycle_writer(GateWal::default(), DatasetVersion(0), 8); + let (_exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let before = log.table().current_version(); + let reserved = LocalMap::prepare(Width::W64, &[1, RESERVED_ID]).unwrap(); + assert!(matches!( + log.advance(1, 1, vec![(4, Some(reserved))]), + Err(AdvanceError::ReservedId(4)) + )); + assert!(matches!( + log.advance(1, 1, vec![(4, None), (4, map(Width::W64, &[1]))]), + Err(AdvanceError::DuplicateTile(4)) + )); + assert_eq!(log.table().current_version(), before); + // The silence twin: a valid advance is claimed and applied. + let v = log.advance(1, 1, vec![(4, map(Width::W64, &[1]))]).unwrap(); + assert_eq!(v.0, before.0 + 1); + while log.settle_next().await.is_some() {} + assert_eq!(answer(log.table(), v.0, 4), Ok((64, vec![1]))); + }); } -/// Frames are witnesses of one table instance, so a frame sealed before a -/// restore is foreign to the restored table. -#[tokio::test(flavor = "multi_thread")] -async fn a_frame_from_before_a_restore_is_foreign() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - let (mut t, mut log) = LanceMapLog::create(&uri, 1, 1).await.unwrap(); - log.advance(&mut t, 1, 1, vec![(1, map(Width::W64, &[1]))]) - .await - .unwrap(); - let frame = t.seal(); - assert!(t.map(&frame, 1).is_ok()); - let (r, _) = LanceMapLog::restore(&uri).await.unwrap(); - assert!(matches!( - r.map(&frame, 1), - Err(MapRead::Stale(FrameRefusal::ForeignFrame)) - )); - assert!(r.map(&r.seal(), 1).is_ok()); +#[tokio::test] +async fn a_restored_log_continues_and_restores_again() { + let (writer, handle) = cycle_writer(GateWal::default(), DatasetVersion(0), 8); + let (exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); + while log.settle_next().await.is_some() {} + }); + let sink = exit.sink; + let restored = restore(&sink, OWNER, VersionClock::new()).await.unwrap(); + let head = DatasetVersion(sink.head); + let (writer, handle) = cycle_writer(sink, head, 8); + let (exit, first) = tokio::join!(writer.run(), async move { + let mut log = MapLog::resume(restored, handle, OWNER); + let v = log.advance(1, 1, vec![(1, None)]).unwrap(); + assert_eq!(v, RouteVersion(2)); + assert_eq!(applied(log.settle_next().await), v); + log + }); + let again = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + assert_eq!(again.last, RouteVersion(2)); + assert_same(first.table(), &again.table, 2); } -#[tokio::test(flavor = "multi_thread")] -async fn create_refuses_an_existing_dataset() { - let dir = tempfile::tempdir().unwrap(); - let uri = dir.path().join("map.lance").to_string_lossy().into_owned(); - LanceMapLog::create(&uri, 1, 1).await.unwrap(); +#[tokio::test] +async fn a_frame_from_before_a_restore_is_foreign() { + let (writer, handle) = cycle_writer(GateWal::default(), DatasetVersion(0), 8); + let (exit, log) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); + while log.settle_next().await.is_some() {} + log + }); + let frame = log.table().seal(); + let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); assert!(matches!( - LanceMapLog::create(&uri, 1, 1).await.unwrap_err(), - StoreError::StaleWriter { - expected: None, - found: Some(1) - } + restored.table.map(&frame, 1), + Err(MapRead::Stale(FrameRefusal::ForeignFrame)) )); + // The same version still answers through diagnose_at. + assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![1]))); } From 7e8802ba02669a58d983a2eb73320d17e7214c14 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:57:41 +0000 Subject: [PATCH 07/14] dt64-lance: map log on the fire-and-forget cycle writer, green 12 tests: the round trip on LanceCycleWriter answers every past version exactly (one Lance version per cycle); gated in-memory interleavings cover submit-before-seal, a clock tick between claim and seal, a failed seal followed by a poisoned one, ownership on every landing, refusals before a claim, resume + restore again, and foreign frames after a restore. Requires lance-graph's ndarray-hpc feature (its fallback does not build). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/Cargo.toml | 5 +- crates/lance-graph-dt64-lance/src/lib.rs | 20 +++++-- .../lance-graph-dt64-lance/tests/map_log.rs | 53 ++++++++++++------- 3 files changed, 53 insertions(+), 25 deletions(-) diff --git a/crates/lance-graph-dt64-lance/Cargo.toml b/crates/lance-graph-dt64-lance/Cargo.toml index 3d8caa4ea..fe2518f03 100644 --- a/crates/lance-graph-dt64-lance/Cargo.toml +++ b/crates/lance-graph-dt64-lance/Cargo.toml @@ -27,8 +27,9 @@ lance-graph-dt64 = { path = "../lance-graph-dt64" } lance-graph-contract = { path = "../lance-graph-contract" } lance-graph-planner = { path = "../lance-graph-planner" } lance-graph-supervisor = { path = "../lance-graph-supervisor", features = ["cycle-driver"] } -# LanceCycleWriter, the WAL (graph::cycle_sink, behind `planner`). -lance-graph = { path = "../lance-graph", default-features = false, features = ["planner"] } +# LanceCycleWriter, the WAL (graph::cycle_sink, behind `planner`). `ndarray-hpc` +# is required: lance-graph does not build without it (its fallback is stale). +lance-graph = { path = "../lance-graph", default-features = false, features = ["planner", "ndarray-hpc"] } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index 38f554d6e..2bf3da7cd 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -186,6 +186,13 @@ impl MapLog { &self.table } + /// Give up the log and keep its table. Drops this log's writer handle, so + /// the writer stops once every other handle is gone too; anything still + /// pending is not applied. + pub fn into_table(self) -> MapTable { + self.table + } + /// Submitted advances whose seal has not been settled yet. pub fn pending(&self) -> usize { self.pending.len() @@ -452,7 +459,10 @@ pub async fn restore( } /// A decoded marker: `(kind, version, source, relation_map)`. -fn decode_marker(cycle: u64, image: &BTreeMap>) -> Result<(u8, u64, u64, u64), ReplayError> { +fn decode_marker( + cycle: u64, + image: &BTreeMap>, +) -> Result<(u8, u64, u64, u64), ReplayError> { let p = image .get(&MARKER_ROW) .ok_or(ReplayError::MissingMarker { cycle })?; @@ -557,8 +567,8 @@ fn changes_of( let mut globals = Vec::new(); let mut ended = false; for (_, p) in &rows { - for slot in p.chunks_exact(8) { - let id = u64::from_le_bytes(slot.try_into().expect("8 bytes")); + for slot in p.as_chunks::<8>().0 { + let id = u64::from_le_bytes(*slot); if id == RESERVED_ID { ended = true; } else if ended { @@ -644,7 +654,9 @@ mod tests { p }; let replay_of = |cycles: Vec<(u64, BTreeMap>)>| { - replay(&cycles, VersionClock::new()).map(|_| ()).unwrap_err() + replay(&cycles, VersionClock::new()) + .map(|_| ()) + .unwrap_err() }; assert_eq!(replay_of(vec![]), ReplayError::Empty); assert_eq!( diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs index d83bef33a..a87e69524 100644 --- a/crates/lance-graph-dt64-lance/tests/map_log.rs +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -77,6 +77,9 @@ fn history() -> Vec)>> { ] } +/// One sealed cycle as the fake keeps it: id, landings, coalesced image. +type SealedCycleRecord = (CycleId, Vec, BTreeMap>); + /// One-writer in-memory sink. Fences on `base_version`, publishes head + 1, /// keeps each cycle's landings and image, can hold commits behind a gate and /// can fail a chosen cycle. Like the real writer, scanned landings carry no @@ -84,7 +87,7 @@ fn history() -> Vec)>> { #[derive(Default)] struct GateWal { head: u64, - cycles: Vec<(CycleId, Vec, BTreeMap>)>, + cycles: Vec, gate: Option>, fail_on: Option, } @@ -163,7 +166,9 @@ fn applied(s: Option) -> RouteVersion { async fn a_restored_table_answers_every_past_version_exactly() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("map.lance"); - let sink = LanceCycleWriter::open(path.to_str().unwrap()).await.unwrap(); + let sink = LanceCycleWriter::open(path.to_str().unwrap()) + .await + .unwrap(); let head = sink.head(); let (writer, handle) = cycle_writer(sink, head, 8); @@ -173,18 +178,19 @@ async fn a_restored_table_answers_every_past_version_exactly() { let v = log.advance(1, 2 + k as u64, changes).unwrap(); assert_eq!(v, RouteVersion(k as u64 + 1)); } - assert!(log.settle_next().await.is_none(), "genesis applies nothing"); for k in 1..=6 { assert_eq!(applied(log.settle_next().await), RouteVersion(k)); } - log + log.into_table() }); - let live = live.table(); + let live = &live; // One Lance version per cycle: genesis plus six advances. assert_eq!(exit.sink.head(), DatasetVersion(head.0 + 7)); - let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + let restored = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); assert_eq!(restored.last, RouteVersion(6)); assert_same(live, &restored.table, 6); for tile in TILES { @@ -208,7 +214,9 @@ async fn the_owner_returns_before_the_seal_and_applies_only_after_it() { let (_exit, ()) = tokio::join!(writer.run(), async move { let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); let v1 = log.advance(1, 1, vec![(1, map(Width::W64, &[5]))]).unwrap(); - let v2 = log.advance(1, 1, vec![(1, map(Width::W64, &[5, 6]))]).unwrap(); + let v2 = log + .advance(1, 1, vec![(1, map(Width::W64, &[5, 6]))]) + .unwrap(); assert_eq!((v1, v2), (RouteVersion(1), RouteVersion(2))); // Nothing sealed yet: the table has not moved. @@ -224,7 +232,6 @@ async fn the_owner_returns_before_the_seal_and_applies_only_after_it() { ); gate.add_permits(3); - assert!(log.settle_next().await.is_none()); assert_eq!(applied(log.settle_next().await), v1); assert_eq!(applied(log.settle_next().await), v2); assert_eq!(answer(log.table(), 2, 1), Ok((64, vec![5, 6]))); @@ -249,14 +256,15 @@ async fn a_clock_tick_between_claim_and_seal_lands_at_the_claimed_version() { clock.tick(); clock.tick(); gate.add_permits(2); - assert!(log.settle_next().await.is_none()); assert_eq!(applied(log.settle_next().await), v); assert_eq!(log.table().current_version(), RouteVersion(v.0 + 2)); assert_eq!(answer(log.table(), v.0, 1), Ok((64, vec![9]))); assert_eq!(answer(log.table(), v.0 - 1, 1), Err(MapRead::NoSuchTile(1))); }); - let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + let restored = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); assert_eq!(restored.last, RouteVersion(1)); assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![9]))); } @@ -274,11 +282,12 @@ async fn a_failed_seal_applies_nothing_and_poisons_what_follows() { let (exit, ()) = tokio::join!(writer.run(), async move { let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); let v1 = log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); - let v2 = log.advance(1, 1, vec![(1, map(Width::W64, &[1, 2]))]).unwrap(); + let v2 = log + .advance(1, 1, vec![(1, map(Width::W64, &[1, 2]))]) + .unwrap(); let v3 = log.advance(1, 1, vec![(2, map(Width::W64, &[3]))]).unwrap(); gate.add_permits(4); - assert!(log.settle_next().await.is_none()); assert_eq!(applied(log.settle_next().await), v1); match log.settle_next().await { Some(Settled::NotDurable { @@ -314,7 +323,9 @@ async fn a_failed_seal_applies_nothing_and_poisons_what_follows() { }); // The store holds genesis and v1 only, and replays to the same table. - let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + let restored = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); assert_eq!(restored.last, RouteVersion(1)); assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![1]))); assert_eq!(answer(&restored.table, 1, 2), Err(MapRead::NoSuchTile(2))); @@ -385,11 +396,13 @@ async fn a_restored_log_continues_and_restores_again() { let v = log.advance(1, 1, vec![(1, None)]).unwrap(); assert_eq!(v, RouteVersion(2)); assert_eq!(applied(log.settle_next().await), v); - log + log.into_table() }); - let again = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + let again = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); assert_eq!(again.last, RouteVersion(2)); - assert_same(first.table(), &again.table, 2); + assert_same(&first, &again.table, 2); } #[tokio::test] @@ -399,10 +412,12 @@ async fn a_frame_from_before_a_restore_is_foreign() { let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); while log.settle_next().await.is_some() {} - log + log.into_table() }); - let frame = log.table().seal(); - let restored = restore(&exit.sink, OWNER, VersionClock::new()).await.unwrap(); + let frame = log.seal(); + let restored = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); assert!(matches!( restored.table.map(&frame, 1), Err(MapRead::Stale(FrameRefusal::ForeignFrame)) From 542dd6beceb0b977f7d7378d959797363a15d03b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 19:59:56 +0000 Subject: [PATCH 08/14] board: DR map table entry and D-DT64-7 plan follow the fire-and-forget rebuild Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../2026-11-10-dt64-map-table-lance.md | 69 ++++++++++++------- ...-7-behaviour-addressing-registration-v1.md | 2 +- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.claude/board/entries/2026-11-10-dt64-map-table-lance.md b/.claude/board/entries/2026-11-10-dt64-map-table-lance.md index 4e3c6d613..7249e616e 100644 --- a/.claude/board/entries/2026-11-10-dt64-map-table-lance.md +++ b/.claude/board/entries/2026-11-10-dt64-map-table-lance.md @@ -1,34 +1,51 @@ -# 2026-11-10 — DR map table stored as a separate Lance-versioned dataset +# 2026-11-10 — DR map table made durable through the fire-and-forget cycle writer -**Status:** VERIFIED-IN-CODE (tests on a real Lance 13 dataset). Answers the -storage half of D-DT64-7 O5 / D10' (D-LITHO Q5: "which table"). +**Status:** VERIFIED-IN-CODE (12 tests; the round trip runs on a real +`LanceCycleWriter`). Answers the storage half of D-DT64-7 O5 / D10' +(D-LITHO Q5: "which table"). WAL choice: D-DR64-WAL-1 (EXO-225). -**What.** `crates/lance-graph-dt64-lance` (excluded, own workspace, like -`lance-graph-dir-sim-lance`) stores `dr::MapTable` in Lance. dt64 stays free of -lance and arrow. +> ⊘ **SUPERSEDED the same day:** the first version of this crate wrote its +> own Lance dataset synchronously, compared dataset versions to refuse a +> stale writer, and reported a post-commit `Race` / `ClockMoved`. That is a +> custom WAL with arbitration, which the persistence doctrine forbids +> (`.claude/v3/knowledge/d-mbx-a6-owner-consume-and-persistence.md`). It was +> rebuilt on the cycle writer. -- One `MapTable::advance` is one Lance commit. Rows are written first, then - the same advance is applied to the table; a refused or failed commit leaves - the table unchanged. -- One row per local position (D10'), plus genesis, advance and removal - markers. Ids only. -- Two clocks, never mixed: the DR route version is in the rows; the Lance - version is the physical commit counter (stale writer, race, and - `restore_at(k)` via `checkout_version`). -- `restore` replays every advance at its own route version, so an extension - stays an extension (generation and event counts equal), a replacement a new - generation, a removal an event. Route-version gaps from a shared clock - replay exactly. Rows that are not a log are refused. +**What.** `crates/lance-graph-dt64-lance` (excluded, own workspace) makes +`dr::MapTable` durable. dt64 stays free of lance and arrow. -**Tests.** 4 unit + 6 integration. Disable runs, each restored: -stale-writer check removed → `a_stale_writer_writes_nothing` and -`create_refuses_an_existing_dataset` fail; gap ticks removed → -`route_versions_with_gaps_replay_as_route_versions` fails. +- `MapLog::advance` claims the route version (`MapTable::claim`), turns the + advance into casts on behalf of the owning mailbox, submits them with + `CycleWriterHandle::try_submit` (cycle id = route version) and returns at + once. One background task owns the `LanceCycleWriter`. +- `MapLog::settle` applies an advance with `MapTable::advance_at` only when + its seal ticket is `Ok`. A failed seal applies nothing and its version is a + gap; every later advance is refused (`Poisoned`), matching the writer, + which seals nothing after a failure. +- One 512-byte cast per 64 ids: a W64 tile is one cast, a W256 tile four, a + removal one; each advance adds one marker cast (version, source, + relation-map revisions). +- `restore` reads every sealed cycle (`scan_sealed` + `scan_image`) and + re-applies each advance at its own route version. It refuses a landing on + behalf of another mailbox and any image that is not a valid log. +- There is no arbitration: one owner holds the `MapLog`, which owns its + table mutably. Two writers on one dataset are outside the contract + (`LanceCycleWriter` fences on its own cached head). + +**Tests.** 4 unit + 8 integration. Disable runs, each restored: +- applying without a sealed ticket → `a_failed_seal_applies_nothing_and_poisons_what_follows`; +- no owner check on restore → `every_landing_is_on_behalf_of_the_owner`; +- applying at the clock's current version instead of the claimed one → 4 tests, + including `a_clock_tick_between_claim_and_seal_lands_at_the_claimed_version`; +- no poison gate → the failed-seal test; +- no reserved-id check → `a_reserved_id_or_a_repeated_tile_is_refused_before_a_version_is_claimed`. **Open.** - Frames are not restored: a frame is one table instance's witness, so a - frame from before a restore is `ForeignFrame` (pinned by a test). Executing - at a past version after a restore needs a frame issued at that version — - a DR owner decision. + frame from before a restore is `ForeignFrame` (pinned by a test). Issuing a + frame at a past version after a restore is a DR owner decision. - A table whose clock is shared with a `RouteLog` needs the route log - stored too; only the map is stored here. + durable too; only the map is stored here. +- The map table's compaction schedule (L's files and scan time grow with + advances; EXO-225). +- lance-graph does not build without `ndarray-hpc`; this crate enables it. diff --git a/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md b/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md index d91dbf441..94eeded85 100644 --- a/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md +++ b/.claude/plans/d-dt64-7-behaviour-addressing-registration-v1.md @@ -120,7 +120,7 @@ Preserved throughout: the `0xC401` name-bound reading, the `FnIndex` opcode cont **Status:** core shipped (`dr::MapTable`, T1–T4 green, both disable runs red). Stored selections wait on O4. -**Storage (2026-11-10):** the versioned table is stored as a separate Lance dataset by `crates/lance-graph-dt64-lance` (`LanceMapLog`). One `MapTable::advance` is one Lance commit; one row per local position; route versions in the rows, the Lance version only the commit counter; `restore` replays every advance exactly and `restore_at(k)` reads the table as of a past commit. OPEN: frames are not restored (a frame is one table instance's witness), so executing at a past version after a restore needs a DR decision; a table sharing its clock with a `RouteLog` needs the route log stored too. +**Storage (2026-11-10):** the versioned table is made durable by `crates/lance-graph-dt64-lance` (`MapLog`) through the fire-and-forget cycle writer over `LanceCycleWriter` (D-DR64-WAL-1). An advance claims its route version, is submitted as casts on behalf of the owning mailbox, and is applied with `advance_at` only after its seal ticket is `Ok`; a failed seal is a gap. `restore` replays every sealed cycle at its own route version. (⊘ An earlier same-day version wrote its own dataset synchronously with stale-writer/race checks; replaced.) OPEN: frames are not restored (a frame is one table instance's witness), so executing at a past version after a restore needs a DR decision; a table sharing its clock with a `RouteLog` needs the route log stored too. O5 is a BindStore plan item (D-DT64-6 Amendment A). Litho is the mental model it came from, not a component. From 0a89ed9f69451cfb9e7a62d69a12a2185fec7243 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:06:22 +0000 Subject: [PATCH 09/14] dt64-lance: replay at the stored genesis; keep a pending advance across a cancelled settle Codex P2s on #1993. - restore built the table with MapTable::with_clock, which takes its last version from the clock, so a shared clock that had moved on made a valid log fail with NotAfterLast. New MapTable::with_clock_at starts the table at the stored genesis and keeps the clock for later issuance. - settle_next popped the pending advance before awaiting its seal, so a cancelled wait dropped a durable advance. New SealTicket::wait(&mut self) waits without consuming the ticket; the entry stays queued until its result is in. Tests: a_restore_onto_a_clock_that_has_moved_on_replays_exactly, a_cancelled_settle_keeps_the_advance_pending, and a dt64 unit test for with_clock_at. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/Cargo.toml | 2 +- crates/lance-graph-dt64-lance/src/lib.rs | 20 ++++-- .../lance-graph-dt64-lance/tests/map_log.rs | 62 +++++++++++++++++++ crates/lance-graph-dt64/src/dr.rs | 43 +++++++++++++ crates/lance-graph-dt64/tests/dr.rs | 32 ++++++++++ .../src/cycle_writer.rs | 8 +++ 6 files changed, 160 insertions(+), 7 deletions(-) diff --git a/crates/lance-graph-dt64-lance/Cargo.toml b/crates/lance-graph-dt64-lance/Cargo.toml index fe2518f03..441276049 100644 --- a/crates/lance-graph-dt64-lance/Cargo.toml +++ b/crates/lance-graph-dt64-lance/Cargo.toml @@ -32,5 +32,5 @@ lance-graph-supervisor = { path = "../lance-graph-supervisor", features = ["cycl lance-graph = { path = "../lance-graph", default-features = false, features = ["planner", "ndarray-hpc"] } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } tempfile = "3" diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index 2bf3da7cd..410049fca 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -252,12 +252,13 @@ impl MapLog { /// calls [`MapLog::settle`]. pub async fn settle_next(&mut self) -> Option { loop { + // The entry stays queued while waiting: if this future is + // cancelled, the advance is still pending and a later settle + // applies it. + let result = self.pending.front_mut()?.ticket.wait().await; let Pending { - version, - advance, - ticket, - } = self.pending.pop_front()?; - let result = ticket.sealed().await; + version, advance, .. + } = self.pending.pop_front().expect("front exists"); if let Some(s) = self.resolve(version, advance, result) { return Some(s); } @@ -499,7 +500,14 @@ fn replay( while clock.current().0 < version { clock.tick(); } - let mut table = MapTable::with_clock(source, relation_map, clock.clone()); + // Start at the stored genesis, not at the clock: a shared clock may + // already be past it. + let mut table = + MapTable::with_clock_at(source, relation_map, clock.clone(), RouteVersion(version)) + .map_err(|refusal| ReplayError::Apply { + cycle: *first, + refusal, + })?; let mut last = RouteVersion(version); for (cycle, image) in rest { let (kind, version, source, relation_map) = decode_marker(*cycle, image)?; diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs index a87e69524..327bb23c8 100644 --- a/crates/lance-graph-dt64-lance/tests/map_log.rs +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -425,3 +425,65 @@ async fn a_frame_from_before_a_restore_is_foreign() { // The same version still answers through diagnose_at. assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![1]))); } + +#[tokio::test] +async fn a_restore_onto_a_clock_that_has_moved_on_replays_exactly() { + let (writer, handle) = cycle_writer(GateWal::default(), DatasetVersion(0), 8); + let (exit, live) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + log.advance(1, 1, vec![(1, map(Width::W64, &[4]))]).unwrap(); + log.advance(1, 1, vec![(1, map(Width::W64, &[4, 5]))]) + .unwrap(); + while log.settle_next().await.is_some() {} + log.into_table() + }); + + // Another component sharing the clock was restored first and moved it on. + let shared = VersionClock::new(); + for _ in 0..5 { + shared.tick(); + } + let restored = restore(&exit.sink, OWNER, shared.clone()).await.unwrap(); + assert_eq!(restored.last, RouteVersion(2)); + assert_same(&live, &restored.table, 2); + assert_eq!(restored.table.current_version(), RouteVersion(5)); + + // Advances continue on the shared clock. + let head = DatasetVersion(exit.sink.head); + let (writer, handle) = cycle_writer(exit.sink, head, 8); + let ((), ()) = tokio::join!( + async { + let _ = writer.run().await; + }, + async move { + let mut log = MapLog::resume(restored, handle, OWNER); + let v = log.advance(1, 1, vec![(1, None)]).unwrap(); + assert_eq!(v, RouteVersion(6)); + assert_eq!(applied(log.settle_next().await), v); + } + ); +} + +#[tokio::test] +async fn a_cancelled_settle_keeps_the_advance_pending() { + let gate = Arc::new(Semaphore::new(0)); + let sink = GateWal { + gate: Some(gate.clone()), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + let (_exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let v = log.advance(1, 1, vec![(1, map(Width::W64, &[8]))]).unwrap(); + + // Waiting is cancelled before the seal lands (a shutdown timeout). + let waited = + tokio::time::timeout(std::time::Duration::from_millis(20), log.settle_next()).await; + assert!(waited.is_err(), "the seal is held, so the wait times out"); + assert_eq!(log.pending(), 2, "genesis and the advance are still queued"); + + gate.add_permits(2); + assert_eq!(applied(log.settle_next().await), v); + assert_eq!(answer(log.table(), v.0, 1), Ok((64, vec![8]))); + }); +} diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index c746907ac..50c7ae1dd 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -392,6 +392,16 @@ impl FrameStore { /// A store whose advances tick `clock`. pub fn with_clock(revisions: FrameRevisions, clock: VersionClock) -> Self { let version = clock.current(); + Self::with_clock_at(revisions, clock, version) + } + + /// A store whose current seal is at `version`, which the caller has + /// checked is not past `clock`'s current version. + fn with_clock_at( + revisions: FrameRevisions, + clock: VersionClock, + version: RouteVersion, + ) -> Self { Self { identity: Arc::new(()), current: Arc::new(revisions), @@ -888,6 +898,39 @@ impl MapTable { } } + /// An empty table whose genesis is at `genesis`, an earlier version of + /// `clock` (a table rebuilt from storage while the clock it shares has + /// moved on). Its first advance must come after `genesis`. Refuses a + /// genesis the clock has not issued. + pub fn with_clock_at( + source: u64, + relation_map: u64, + clock: VersionClock, + genesis: RouteVersion, + ) -> Result { + let current = clock.current(); + if genesis > current { + return Err(AdvanceRefusal::NotIssued { + version: genesis, + current, + }); + } + let tiles = BTreeMap::new(); + let coordinates = Self::coordinates(&tiles); + Ok(Self { + frames: FrameStore::with_clock_at( + FrameRevisions { + source, + relation_map, + coordinates, + }, + clock, + genesis, + ), + tiles, + }) + } + /// The clock this table advances (a clone that shares the counter). pub fn clock(&self) -> VersionClock { self.frames.clock.clone() diff --git a/crates/lance-graph-dt64/tests/dr.rs b/crates/lance-graph-dt64/tests/dr.rs index 8aea1ef22..19188b3b0 100644 --- a/crates/lance-graph-dt64/tests/dr.rs +++ b/crates/lance-graph-dt64/tests/dr.rs @@ -848,3 +848,35 @@ fn an_unchanged_space_survives_an_advance_of_another() { assert_eq!(t.code(&f, &priority(), &vk("high")), Ok(1)); assert_eq!(t.key(&f, &priority(), 1), Ok(&vk("high"))); } + +mod genesis_at { + use lance_graph_dt64::dr::{AdvanceRefusal, MapTable, RouteVersion, VersionClock}; + + #[test] + fn a_table_can_start_at_a_past_version_of_its_clock() { + let clock = VersionClock::new(); + for _ in 0..4 { + clock.tick(); + } + let mut t = MapTable::with_clock_at(1, 1, clock.clone(), RouteVersion(1)).unwrap(); + // The first advance after genesis applies, though the clock is at 4. + assert_eq!( + t.advance_at(RouteVersion(2), 1, 1, vec![]), + Ok(RouteVersion(2)) + ); + // Not at or before genesis. + let mut t = MapTable::with_clock_at(1, 1, clock.clone(), RouteVersion(3)).unwrap(); + assert_eq!( + t.advance_at(RouteVersion(3), 1, 1, vec![]), + Err(AdvanceRefusal::NotAfterLast { + last: RouteVersion(3), + version: RouteVersion(3) + }) + ); + // A genesis the clock has not issued is refused. + assert!(matches!( + MapTable::with_clock_at(1, 1, clock, RouteVersion(9)), + Err(AdvanceRefusal::NotIssued { .. }) + )); + } +} diff --git a/crates/lance-graph-supervisor/src/cycle_writer.rs b/crates/lance-graph-supervisor/src/cycle_writer.rs index 67e033d01..add57c72e 100644 --- a/crates/lance-graph-supervisor/src/cycle_writer.rs +++ b/crates/lance-graph-supervisor/src/cycle_writer.rs @@ -144,6 +144,14 @@ impl SealTicket { self.rx.await.unwrap_or(Err(SealError::WriterGone)) } + /// Wait for this cycle's seal without consuming the ticket, so a caller + /// that is cancelled while waiting still holds it and can wait again. + /// After this returns, the ticket is spent: call neither `wait` nor + /// `try_sealed` on it again. + pub async fn wait(&mut self) -> Result { + (&mut self.rx).await.unwrap_or(Err(SealError::WriterGone)) + } + /// The seal if it has already resolved, without waiting. `None` while the /// writer is still on it. After this returns `Some`, the ticket is spent. pub fn try_sealed(&mut self) -> Option> { From 36098aec8d3a745abd8948cacf5729e005cfc6fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:09:31 +0000 Subject: [PATCH 10/14] ci(dt64 map table): also run when the cycle writer, planner sink or LanceCycleWriter change Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .github/workflows/dt64-map-table-lance.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dt64-map-table-lance.yml b/.github/workflows/dt64-map-table-lance.yml index 62f29aeb6..1c6175406 100644 --- a/.github/workflows/dt64-map-table-lance.yml +++ b/.github/workflows/dt64-map-table-lance.yml @@ -1,5 +1,6 @@ name: dt64 map table in Lance -# lance-graph-dt64-lance stores dt64's DR map table (dr::MapTable) in Lance. +# lance-graph-dt64-lance makes dt64's DR map table (dr::MapTable) durable +# through the supervisor's cycle writer over LanceCycleWriter. # It is workspace-excluded (own [workspace] root), so no other job builds it. # Without this one a change to dt64's dr module or a lance bump could break it # unnoticed. @@ -15,6 +16,10 @@ on: - crates/lance-graph-contract/** - crates/lance-graph-dt64/** - crates/lance-graph-dt64-lance/** + # The map log is written through the cycle writer over LanceCycleWriter. + - crates/lance-graph-supervisor/** + - crates/lance-graph-planner/** + - crates/lance-graph/src/graph/cycle_sink.rs - .github/workflows/dt64-map-table-lance.yml workflow_dispatch: From c2ae0b15d003ce1f90e805405a8f2dc3d0fed082 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:19:55 +0000 Subject: [PATCH 11/14] fix(dt64-lance): into_table refuses an undrained log; a refused durable cycle poisons it MapLog::into_table returned the table while sealed cycles were still unsettled, dropping them silently. It now returns Undrained (carrying the log back) while anything is pending or a durable cycle was refused. A durable cycle the table refuses (Settled::Refused) now poisons the log: later advances are refused, as after a failed seal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/src/lib.rs | 153 ++++++++++++++++-- .../lance-graph-dt64-lance/tests/map_log.rs | 41 ++++- 2 files changed, 179 insertions(+), 15 deletions(-) diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index 410049fca..a8e348198 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -50,7 +50,6 @@ use lance_graph_dt64::dr::{ AdvanceRefusal, LocalMap, MapRefusal, MapTable, RouteVersion, TileId, VersionClock, Width, }; use lance_graph_planner::persist_sink::{CycleId, SweepSlot, WalSink, WriteFailed}; -use lance_graph_supervisor::cycle_driver::SealedCycle; use lance_graph_supervisor::cycle_writer::{CycleWriterHandle, SealError, SealTicket, SubmitError}; /// Bytes in one cast payload (the cycle writer's artifact width). @@ -114,7 +113,9 @@ pub enum Settled { /// The cycle is durable but the table refused to apply it. `MapLog` /// applies its own advances in claimed order and owns its table, so its /// API cannot produce this; it is reported rather than dropped so a - /// disagreement between store and table can never pass silently. A + /// disagreement between store and table can never pass silently. The + /// store now holds a cycle the table lacks: the log refuses every later + /// advance ([`AdvanceError::Poisoned`]) and [`MapLog::into_table`]. A /// restore replays the store. Refused { version: RouteVersion, @@ -140,6 +141,43 @@ pub struct MapLog { next_position: u64, pending: VecDeque, failed: Option, + /// The first durable cycle the table refused (see [`Settled::Refused`]). + diverged: Option, +} + +/// [`MapLog::into_table`] refused: the table is not the store's last sealed +/// state. Carries the log back so the caller can settle it, or restore. +pub struct Undrained { + log: Box, +} + +impl Undrained { + /// Submitted cycles not yet settled when the table was asked for. + #[must_use] + pub fn pending(&self) -> usize { + self.log.pending() + } + + /// The durable cycle the table refused, if that is why. + #[must_use] + pub fn diverged(&self) -> Option { + self.log.diverged + } + + /// The log, to settle what is pending and ask again. + #[must_use] + pub fn into_log(self) -> MapLog { + *self.log + } +} + +impl std::fmt::Debug for Undrained { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Undrained") + .field("pending", &self.pending()) + .field("diverged", &self.diverged()) + .finish() + } } impl MapLog { @@ -162,6 +200,7 @@ impl MapLog { next_position: 0, pending: VecDeque::new(), failed: None, + diverged: None, }; let casts = vec![log.marker(GENESIS, version, source, relation_map)]; log.submit(version, None, casts)?; @@ -178,6 +217,7 @@ impl MapLog { next_position: restored.next_position, pending: VecDeque::new(), failed: None, + diverged: None, } } @@ -186,11 +226,24 @@ impl MapLog { &self.table } - /// Give up the log and keep its table. Drops this log's writer handle, so - /// the writer stops once every other handle is gone too; anything still - /// pending is not applied. - pub fn into_table(self) -> MapTable { - self.table + /// Give up the log and keep its table, which must be drained: nothing + /// pending and no durable cycle refused. Otherwise the log comes back in + /// [`Undrained`], so a sealed cycle can never be dropped unapplied + /// (settle it, then ask again). Drops this log's writer handle on + /// success, so the writer stops once every other handle is gone too. + /// + /// # Errors + /// + /// [`Undrained`] while [`MapLog::pending`] is non-zero or a durable + /// cycle was refused ([`Settled::Refused`]). + pub fn into_table(self) -> Result { + if self.pending.is_empty() && self.diverged.is_none() { + Ok(self.table) + } else { + Err(Undrained { + log: Box::new(self), + }) + } } /// Submitted advances whose seal has not been settled yet. @@ -240,7 +293,7 @@ impl MapLog { let Pending { version, advance, .. } = self.pending.pop_front().expect("front exists"); - if let Some(s) = self.resolve(version, advance, result) { + if let Some(s) = self.resolve(version, advance, result.map(|_| ())) { out.push(s); } } @@ -259,7 +312,7 @@ impl MapLog { let Pending { version, advance, .. } = self.pending.pop_front().expect("front exists"); - if let Some(s) = self.resolve(version, advance, result) { + if let Some(s) = self.resolve(version, advance, result.map(|_| ())) { return Some(s); } } @@ -269,7 +322,7 @@ impl MapLog { &mut self, version: RouteVersion, advance: Option, - result: Result, + result: Result<(), SealError>, ) -> Option { match result { Err(error) => { @@ -284,7 +337,11 @@ impl MapLog { .advance_at(version, source, relation_map, changes) { Ok(v) => Settled::Applied(v), - Err(refusal) => Settled::Refused { version, refusal }, + Err(refusal) => { + self.failed.get_or_insert(version); + self.diverged.get_or_insert(version); + Settled::Refused { version, refusal } + } }, ) } @@ -600,6 +657,80 @@ fn changes_of( mod tests { use super::*; + /// Accepts every cycle; enough to run a `MapLog` against a real writer. + struct AcceptAll { + head: u64, + } + + impl WalSink for AcceptAll { + async fn commit_cycle( + &mut self, + batch: lance_graph_planner::persist_sink::DetachedCycleBatch, + ) -> Result< + lance_graph_planner::persist_sink::CommitOutcome, + lance_graph_planner::persist_sink::CommitError, + > { + self.head += 1; + Ok( + lance_graph_planner::persist_sink::CommitOutcome::Committed { + version: lance_graph_contract::scheduler::DatasetVersion(self.head), + cycle: batch.frame.cycle, + batch_hash: batch.batch_hash, + }, + ) + } + + async fn scan_sealed( + &self, + _after: Option, + ) -> Result, WriteFailed> { + Ok(Vec::new()) + } + + async fn timeline( + &self, + ) -> Result, WriteFailed> { + Ok(Vec::new()) + } + } + + /// A durable cycle the table refuses poisons the log: no later advance, + /// no table handed out as if it were the store's state. `MapLog`'s API + /// cannot produce the refusal, so the test resolves an already-applied + /// version a second time, which the table refuses as `NotAfterLast`. + #[tokio::test] + async fn a_durable_cycle_the_table_refuses_poisons_the_log() { + let (writer, handle) = lance_graph_supervisor::cycle_writer::cycle_writer( + AcceptAll { head: 0 }, + lance_graph_contract::scheduler::DatasetVersion(0), + 8, + ); + let (_exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, 7, 1, 1, VersionClock::new()).unwrap(); + let v = log.advance(1, 1, vec![(1, None)]).unwrap(); + assert!(matches!(log.settle_next().await, Some(Settled::Applied(a)) if a == v)); + // Silence twin: a drained, agreeing log hands out its table. + assert_eq!(log.pending(), 0); + assert!(log.diverged.is_none()); + + let refused = log.resolve(v, Some((1, 1, vec![(1, None)])), Ok(())); + assert!( + matches!(refused, Some(Settled::Refused { version, .. }) if version == v), + "{refused:?}" + ); + assert!(matches!( + log.advance(1, 1, vec![]), + Err(AdvanceError::Poisoned { failed }) if failed == v + )); + let undrained = log + .into_table() + .err() + .expect("a diverged log keeps its table"); + assert_eq!(undrained.pending(), 0); + assert_eq!(undrained.diverged(), Some(v)); + }); + } + fn image(entries: &[(u64, Vec)]) -> BTreeMap> { entries.iter().cloned().collect() } diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs index 327bb23c8..40b169492 100644 --- a/crates/lance-graph-dt64-lance/tests/map_log.rs +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -181,7 +181,7 @@ async fn a_restored_table_answers_every_past_version_exactly() { for k in 1..=6 { assert_eq!(applied(log.settle_next().await), RouteVersion(k)); } - log.into_table() + log.into_table().unwrap() }); let live = &live; @@ -396,7 +396,7 @@ async fn a_restored_log_continues_and_restores_again() { let v = log.advance(1, 1, vec![(1, None)]).unwrap(); assert_eq!(v, RouteVersion(2)); assert_eq!(applied(log.settle_next().await), v); - log.into_table() + log.into_table().unwrap() }); let again = restore(&exit.sink, OWNER, VersionClock::new()) .await @@ -412,7 +412,7 @@ async fn a_frame_from_before_a_restore_is_foreign() { let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); log.advance(1, 1, vec![(1, map(Width::W64, &[1]))]).unwrap(); while log.settle_next().await.is_some() {} - log.into_table() + log.into_table().unwrap() }); let frame = log.seal(); let restored = restore(&exit.sink, OWNER, VersionClock::new()) @@ -435,7 +435,7 @@ async fn a_restore_onto_a_clock_that_has_moved_on_replays_exactly() { log.advance(1, 1, vec![(1, map(Width::W64, &[4, 5]))]) .unwrap(); while log.settle_next().await.is_some() {} - log.into_table() + log.into_table().unwrap() }); // Another component sharing the clock was restored first and moved it on. @@ -487,3 +487,36 @@ async fn a_cancelled_settle_keeps_the_advance_pending() { assert_eq!(answer(log.table(), v.0, 1), Ok((64, vec![8]))); }); } + +/// The drained-state contract: `into_table` never hands out a table while a +/// submitted cycle is unsettled, even one the store already sealed. The log +/// comes back, settles, and the second ask returns the applied table. +#[tokio::test] +async fn into_table_refuses_while_a_submitted_cycle_is_unsettled() { + let gate = Arc::new(Semaphore::new(0)); + let sink = GateWal { + gate: Some(gate.clone()), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + + let (exit, table) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let v = log.advance(1, 1, vec![(1, map(Width::W64, &[5]))]).unwrap(); + gate.add_permits(2); + // Let the writer seal both cycles without the log settling them. + for _ in 0..1000 { + tokio::task::yield_now().await; + } + + let undrained = log.into_table().expect_err("sealed but unsettled"); + assert_eq!(undrained.pending(), 2); + assert_eq!(undrained.diverged(), None); + let mut log = undrained.into_log(); + assert_eq!(applied(log.settle_next().await), v); + assert_eq!(log.pending(), 0); + log.into_table().expect("drained") + }); + assert_eq!(exit.sink.cycles.len(), 2, "both cycles were sealed"); + assert_eq!(answer(&table, 1, 1), Ok((64, vec![5]))); +} From a2bc41eb70de09481a0d6818c8fa92be025843a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:20:30 +0000 Subject: [PATCH 12/14] fix(dt64-lance): clippy err_expect in the poison test Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index a8e348198..63f0dc05f 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -724,8 +724,7 @@ mod tests { )); let undrained = log .into_table() - .err() - .expect("a diverged log keeps its table"); + .expect_err("a diverged log keeps its table"); assert_eq!(undrained.pending(), 0); assert_eq!(undrained.diverged(), Some(v)); }); From 8ce17afcfbfb891271e6e1ce73020316de781c54 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:21:39 +0000 Subject: [PATCH 13/14] ci(dt64 map table): check out the sibling ndarray the build path-depends on Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .github/workflows/dt64-map-table-lance.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/dt64-map-table-lance.yml b/.github/workflows/dt64-map-table-lance.yml index 1c6175406..ba19bd1e6 100644 --- a/.github/workflows/dt64-map-table-lance.yml +++ b/.github/workflows/dt64-map-table-lance.yml @@ -19,6 +19,7 @@ on: # The map log is written through the cycle writer over LanceCycleWriter. - crates/lance-graph-supervisor/** - crates/lance-graph-planner/** + - crates/lance-graph/Cargo.toml - crates/lance-graph/src/graph/cycle_sink.rs - .github/workflows/dt64-map-table-lance.yml workflow_dispatch: @@ -50,6 +51,14 @@ jobs: with: path: lance-graph persist-credentials: false + # lance-graph's ndarray-hpc feature and the planner path-depend on + # ../../../ndarray. + - name: Checkout AdaWorldAPI/ndarray (sibling dependency) + uses: actions/checkout@v4 + with: + repository: AdaWorldAPI/ndarray + path: ndarray + persist-credentials: false - name: Install protoc and the protobuf includes (for lance) run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - name: Setup rust toolchain (pinned by rust-toolchain.toml) From 462133860070b281c5cd0576c927121bf32d0ce4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:33:30 +0000 Subject: [PATCH 14/14] fix(dt64-lance): a seal of unknown outcome blocks the table instead of reading as a gap An ambiguous commit (SealRecovery::ResubmitFrozen), a hash conflict (Escalate) or a writer gone before answering may have published the cycle. Settled::OutcomeUnknown reports it; the log is poisoned and into_table refuses until a restore. A proven-unpublished failure stays NotDurable and still drains. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/lance-graph-dt64-lance/src/lib.rs | 42 +++++++++++++--- .../lance-graph-dt64-lance/tests/map_log.rs | 48 +++++++++++++++++++ 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/crates/lance-graph-dt64-lance/src/lib.rs b/crates/lance-graph-dt64-lance/src/lib.rs index 63f0dc05f..3fc663503 100644 --- a/crates/lance-graph-dt64-lance/src/lib.rs +++ b/crates/lance-graph-dt64-lance/src/lib.rs @@ -50,6 +50,7 @@ use lance_graph_dt64::dr::{ AdvanceRefusal, LocalMap, MapRefusal, MapTable, RouteVersion, TileId, VersionClock, Width, }; use lance_graph_planner::persist_sink::{CycleId, SweepSlot, WalSink, WriteFailed}; +use lance_graph_supervisor::cycle_driver::SealRecovery; use lance_graph_supervisor::cycle_writer::{CycleWriterHandle, SealError, SealTicket, SubmitError}; /// Bytes in one cast payload (the cycle writer's artifact width). @@ -110,6 +111,15 @@ pub enum Settled { version: RouteVersion, error: SealError, }, + /// The seal failed without proving whether the cycle was published: an + /// ambiguous commit (`SealRecovery::ResubmitFrozen`), a hash conflict + /// (`Escalate`), or a writer that stopped before answering. The store may + /// hold a cycle the table lacks, so the log refuses every later advance + /// and [`MapLog::into_table`]; a restore replays what the store holds. + OutcomeUnknown { + version: RouteVersion, + error: SealError, + }, /// The cycle is durable but the table refused to apply it. `MapLog` /// applies its own advances in claimed order and owns its table, so its /// API cannot produce this; it is reported rather than dropped so a @@ -123,6 +133,18 @@ pub enum Settled { }, } +/// Whether a failed seal leaves open that the cycle was published. +fn outcome_unknown(error: &SealError) -> bool { + match error { + SealError::Failed(f) => !matches!( + f.recovery(), + SealRecovery::Regenerate | SealRecovery::Permanent + ), + SealError::WriterGone => true, + SealError::Poisoned { .. } | SealError::OutOfOrder { .. } => false, + } +} + /// An advance's `(source, relation_map, changes)`. type Advance = (u64, u64, Vec<(TileId, Option)>); @@ -141,7 +163,9 @@ pub struct MapLog { next_position: u64, pending: VecDeque, failed: Option, - /// The first durable cycle the table refused (see [`Settled::Refused`]). + /// The first cycle the store may hold and the table lacks: refused while + /// durable ([`Settled::Refused`]) or of unknown outcome + /// ([`Settled::OutcomeUnknown`]). diverged: Option, } @@ -158,7 +182,7 @@ impl Undrained { self.log.pending() } - /// The durable cycle the table refused, if that is why. + /// The cycle the store may hold and the table lacks, if that is why. #[must_use] pub fn diverged(&self) -> Option { self.log.diverged @@ -227,15 +251,16 @@ impl MapLog { } /// Give up the log and keep its table, which must be drained: nothing - /// pending and no durable cycle refused. Otherwise the log comes back in + /// pending and no cycle the store may hold missing from the table. Otherwise the log comes back in /// [`Undrained`], so a sealed cycle can never be dropped unapplied /// (settle it, then ask again). Drops this log's writer handle on /// success, so the writer stops once every other handle is gone too. /// /// # Errors /// - /// [`Undrained`] while [`MapLog::pending`] is non-zero or a durable - /// cycle was refused ([`Settled::Refused`]). + /// [`Undrained`] while [`MapLog::pending`] is non-zero, or a cycle the + /// store may hold is missing from the table ([`Settled::Refused`], + /// [`Settled::OutcomeUnknown`]). pub fn into_table(self) -> Result { if self.pending.is_empty() && self.diverged.is_none() { Ok(self.table) @@ -327,7 +352,12 @@ impl MapLog { match result { Err(error) => { self.failed.get_or_insert(version); - Some(Settled::NotDurable { version, error }) + if outcome_unknown(&error) { + self.diverged.get_or_insert(version); + Some(Settled::OutcomeUnknown { version, error }) + } else { + Some(Settled::NotDurable { version, error }) + } } Ok(_) => { let (source, relation_map, changes) = advance?; diff --git a/crates/lance-graph-dt64-lance/tests/map_log.rs b/crates/lance-graph-dt64-lance/tests/map_log.rs index 40b169492..2af48ebe8 100644 --- a/crates/lance-graph-dt64-lance/tests/map_log.rs +++ b/crates/lance-graph-dt64-lance/tests/map_log.rs @@ -90,6 +90,8 @@ struct GateWal { cycles: Vec, gate: Option>, fail_on: Option, + /// Commit this cycle, then answer as if the acknowledgement was lost. + ambiguous_on: Option, } impl WalSink for GateWal { @@ -123,6 +125,13 @@ impl WalSink for GateWal { .collect(); self.cycles .push((batch.frame.cycle, landings, batch.image.clone())); + if Some(batch.frame.cycle) == self.ambiguous_on { + return Err(CommitError::Ambiguous { + cycle: batch.frame.cycle, + batch_hash: batch.batch_hash, + cause: "acknowledgement lost".into(), + }); + } Ok(CommitOutcome::Committed { version: DatasetVersion(self.head), cycle: batch.frame.cycle, @@ -320,6 +329,9 @@ async fn a_failed_seal_applies_nothing_and_poisons_what_follows() { other => panic!("expected Poisoned, got {other:?}"), } assert_eq!(log.table().current_version(), before, "nothing was claimed"); + // A proven gap leaves store and table in agreement: the log drains. + log.into_table() + .expect("a proven-unpublished failure still drains"); }); // The store holds genesis and v1 only, and replays to the same table. @@ -520,3 +532,39 @@ async fn into_table_refuses_while_a_submitted_cycle_is_unsettled() { assert_eq!(exit.sink.cycles.len(), 2, "both cycles were sealed"); assert_eq!(answer(&table, 1, 1), Ok((64, vec![5]))); } + +/// A seal whose outcome is unknown may have published the cycle. The log +/// must not hand out a table that may lack it; a restore shows the store did +/// hold it. +#[tokio::test] +async fn an_ambiguous_seal_blocks_the_table_until_a_restore() { + let sink = GateWal { + ambiguous_on: Some(CycleId(1)), + ..GateWal::default() + }; + let (writer, handle) = cycle_writer(sink, DatasetVersion(0), 8); + + let (exit, ()) = tokio::join!(writer.run(), async move { + let mut log = MapLog::create(handle, OWNER, 1, 1, VersionClock::new()).unwrap(); + let v1 = log.advance(1, 1, vec![(1, map(Width::W64, &[4]))]).unwrap(); + match log.settle_next().await { + Some(Settled::OutcomeUnknown { version, .. }) => assert_eq!(version, v1), + other => panic!("expected an unknown outcome, got {other:?}"), + } + assert_eq!(answer(log.table(), 1, 1), Err(MapRead::NoSuchTile(1))); + assert!(matches!( + log.advance(1, 1, vec![]), + Err(AdvanceError::Poisoned { failed }) if failed == v1 + )); + let undrained = log.into_table().expect_err("the table may lack v1"); + assert_eq!(undrained.pending(), 0); + assert_eq!(undrained.diverged(), Some(v1)); + }); + + // The store did hold v1: the restored table has it. + let restored = restore(&exit.sink, OWNER, VersionClock::new()) + .await + .unwrap(); + assert_eq!(restored.last, RouteVersion(1)); + assert_eq!(answer(&restored.table, 1, 1), Ok((64, vec![4]))); +}