From c99157021701cab5159df006cc75cf951cbf38b5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:12:00 +0300 Subject: [PATCH 01/64] chore(storage): update primitives module This change updates the primitives module within the storage crate to align with recent internal refactoring efforts. It ensures consistency across the storage layer's fundamental building blocks. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/lib.rs | 0 crates/tinyflows-storage/src/primitives/mod.rs | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 crates/tinyflows-storage/src/lib.rs create mode 100644 crates/tinyflows-storage/src/primitives/mod.rs diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/tinyflows-storage/src/primitives/mod.rs b/crates/tinyflows-storage/src/primitives/mod.rs new file mode 100644 index 00000000..e69de29b From 2d4be1f65fe606994eb8989e83247a1383205200 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:12:17 +0300 Subject: [PATCH 02/64] feat(primitives): implement core storage primitives This introduces the fundamental building blocks for storage operations, including counters, key-value storage, leases, and repository management. These primitives establish the necessary low-level abstractions for the storage layer. Auto-committed-on: dragonfly --- .../src/primitives/counters.rs | 128 +++++ crates/tinyflows-storage/src/primitives/kv.rs | 146 +++++ .../src/primitives/leases.rs | 523 ++++++++++++++++++ .../tinyflows-storage/src/primitives/repo.rs | 352 ++++++++++++ 4 files changed, 1149 insertions(+) create mode 100644 crates/tinyflows-storage/src/primitives/counters.rs create mode 100644 crates/tinyflows-storage/src/primitives/kv.rs create mode 100644 crates/tinyflows-storage/src/primitives/leases.rs create mode 100644 crates/tinyflows-storage/src/primitives/repo.rs diff --git a/crates/tinyflows-storage/src/primitives/counters.rs b/crates/tinyflows-storage/src/primitives/counters.rs new file mode 100644 index 00000000..25dc5d2e --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/counters.rs @@ -0,0 +1,128 @@ +//! [`Counters`]: named monotonic counters. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use super::COUNTERS; +use super::repo::{Repo, RetryPolicy}; +use crate::document::DocumentStore; +use crate::error::{Result, StorageError}; +use crate::key::Key; + +/// The largest value every driver can store (`i64::MAX`). +const MAX_VALUE: u64 = i64::MAX as u64; + +#[derive(Debug, Serialize, Deserialize)] +struct Body { + value: u64, +} + +/// Atomic counters over [`COUNTERS`], one per `(owner, name)`. +/// +/// Each step is one compare-and-swap [`update`](Repo::update), so two tasks +/// (or two processes on a shared backend) never receive the same number. +/// Counts live in a JSON number, so every driver stores the same range: 0 to +/// `i64::MAX`. [`next`](Self::next) and [`set_at_least`](Self::set_at_least) +/// refuse to go past it with `InvalidInput` rather than store a value some +/// backend could not hold. +#[derive(Debug, Clone)] +pub struct Counters { + repo: Repo, +} + +impl Counters { + /// Counters on `docs`, which must be bound to the scope that owns them. + /// + /// # Panics + /// + /// Never: the collection name is a valid constant. + #[must_use] + #[allow(clippy::expect_used, reason = "COUNTERS is a valid constant name")] + pub fn new(docs: &Arc) -> Self { + Self { + repo: Repo::new(docs, COUNTERS).expect("tsd.counters is a valid collection"), + } + } + + /// Replace the retry policy of the underlying compare-and-swap loop. + #[must_use] + pub fn with_retry(mut self, retry: RetryPolicy) -> Self { + self.repo = self.repo.with_retry(retry); + self + } + + /// Advance the counter and return the new value. The first call returns 1. + /// + /// # Errors + /// + /// A storage error, a conflict past the retry budget, or `InvalidInput` + /// when the counter would pass `i64::MAX`. + pub async fn next(&self, owner: &str, name: &str) -> Result { + let id = id(owner, name)?; + let stored = self + .repo + .update(&id, |body| { + let value = body.map_or(Some(1), |b| { + b.value.checked_add(1).filter(|v| *v <= MAX_VALUE) + }); + value + .map(|value| Some(Body { value })) + .ok_or_else(|| StorageError::invalid_input("counter overflow")) + }) + .await?; + stored + .map(|(body, _)| body.value) + .ok_or_else(unreachable_declined) + } + + /// The current value, 0 for a counter never advanced. + /// + /// # Errors + /// + /// A storage error. + pub async fn peek(&self, owner: &str, name: &str) -> Result { + let found = self.repo.get(&id(owner, name)?).await?; + Ok(found.map_or(0, |found| found.doc.value)) + } + + /// Raise the counter to at least `minimum` and return the resulting value. + /// A counter already at or above it is left alone: use it after importing + /// ids from elsewhere so [`next`](Self::next) cannot hand one out again. + /// + /// # Errors + /// + /// `InvalidInput` when `minimum` is above `i64::MAX`, a storage error, or a + /// conflict past the retry budget. + pub async fn set_at_least(&self, owner: &str, name: &str, minimum: u64) -> Result { + if minimum > MAX_VALUE { + return Err(StorageError::invalid_input("counter value out of range")); + } + let id = id(owner, name)?; + let stored = self + .repo + .update(&id, |body| { + Ok(match body { + Some(b) if b.value >= minimum => None, + _ => Some(Body { value: minimum }), + }) + }) + .await?; + match stored { + Some((body, _)) => Ok(body.value), + None => self.peek(owner, name).await, + } + } +} + +fn id(owner: &str, name: &str) -> Result { + Ok(Key::of([owner, name])?.as_str().to_owned()) +} + +fn unreachable_declined() -> StorageError { + StorageError::backend("counter update declined unexpectedly") +} + +#[cfg(test)] +#[path = "counters_tests.rs"] +mod tests; diff --git a/crates/tinyflows-storage/src/primitives/kv.rs b/crates/tinyflows-storage/src/primitives/kv.rs new file mode 100644 index 00000000..1f0c64ef --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/kv.rs @@ -0,0 +1,146 @@ +//! [`Kv`]: a small key-value store per `(owner, namespace)`. + +use std::sync::Arc; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::KV; +use super::repo::Repo; +use crate::document::{DocumentStore, Precondition, Query, Version}; +use crate::error::{Result, StorageError}; +use crate::filter::{Filter, Sort}; +use crate::key::Key; + +/// Entries per page while [`Kv::list`] walks a namespace. +const PAGE: usize = 200; + +/// A stored entry. Document bodies must be JSON objects, so a value that is a +/// number, string or array cannot be the body itself; the entry carries its +/// coordinates next to the value instead, which also lets the +/// `(owner, ns, key)` index list a namespace in key order. +#[derive(Debug, Serialize, Deserialize)] +struct Entry { + owner: String, + ns: String, + key: String, + value: Value, +} + +/// Key-value entries of one `(owner, namespace)` over [`KV`]. +/// +/// Values are any serde type; they are stored as JSON under the `value` field +/// of the entry document. +#[derive(Debug, Clone)] +pub struct Kv { + repo: Repo, + docs: Arc, + owner: String, + ns: String, +} + +impl Kv { + /// The namespace `ns` of `owner` on `docs`, which must be bound to the + /// scope that owns the data. + /// + /// # Errors + /// + /// `InvalidInput` for an empty owner or namespace. + pub fn ns( + docs: &Arc, + owner: impl Into, + ns: impl Into, + ) -> Result { + let (owner, ns) = (owner.into(), ns.into()); + if owner.is_empty() || ns.is_empty() { + return Err(StorageError::invalid_input( + "a key-value owner and namespace must not be empty", + )); + } + Ok(Self { + repo: Repo::new(docs, KV)?, + docs: Arc::clone(docs), + owner, + ns, + }) + } + + fn id(&self, key: &str) -> Result { + Ok(Key::of([self.owner.as_str(), self.ns.as_str(), key])? + .as_str() + .to_owned()) + } + + /// Read `key`. + /// + /// # Errors + /// + /// A storage error, or `Serialization` when the value is not a `T`. + pub async fn get(&self, key: &str) -> Result> { + match self.repo.get(&self.id(key)?).await? { + Some(found) => Ok(Some(serde_json::from_value(found.doc.value)?)), + None => Ok(None), + } + } + + /// Store `value` under `key`, replacing any earlier one. + /// + /// # Errors + /// + /// A storage error, or `Serialization`. + pub async fn put(&self, key: &str, value: &T) -> Result { + let entry = Entry { + owner: self.owner.clone(), + ns: self.ns.clone(), + key: key.to_owned(), + value: serde_json::to_value(value)?, + }; + self.repo.put(&self.id(key)?, &entry).await + } + + /// Remove `key` and report whether it existed. + /// + /// # Errors + /// + /// A storage error. + pub async fn delete(&self, key: &str) -> Result { + self.docs + .delete(KV, &self.id(key)?, Precondition::None) + .await + } + + /// Every entry whose key starts with `prefix` (all entries for `""`), in + /// key order, paging through the namespace internally. + /// + /// # Errors + /// + /// A storage error, or `Serialization` when a value is not a `T`. + pub async fn list(&self, prefix: &str) -> Result> { + let mut filter = + Filter::eq("owner", self.owner.as_str()).and(Filter::eq("ns", self.ns.as_str())); + if !prefix.is_empty() { + filter = filter.and(Filter::gte("key", prefix)); + } + let mut query = Query::filter(filter).sort(Sort::asc("key")).limit(PAGE); + let mut out = Vec::new(); + loop { + let page = self.repo.query(&query).await?; + for item in page.items { + if !item.doc.key.starts_with(prefix) { + // Sorted by key: the first miss past the lower bound ends the range. + return Ok(out); + } + out.push((item.doc.key, serde_json::from_value(item.doc.value)?)); + } + match page.next { + Some(cursor) => query = query.after(cursor), + None => return Ok(out), + } + } + } +} + +#[cfg(test)] +#[path = "kv_tests.rs"] +mod tests; diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs new file mode 100644 index 00000000..61a08536 --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -0,0 +1,523 @@ +//! [`Leases`]: time-limited ownership with a fencing epoch. +//! +//! # Why expiry is a body field +//! +//! A lease could carry its expiry as record metadata +//! ([`WriteOptions::expires_ms`](crate::WriteOptions)), which would hide it +//! once it ran out. It deliberately does not. A takeover must read the expired +//! record to learn its `epoch` and write `epoch + 1`; if the record vanished, +//! the next holder would restart at epoch 1 and a stale holder's +//! [`Fence`] over epoch 1 would hold again. So the record stays, its +//! `expires_ms` field says when the lease lapses, and `epoch` only ever moves +//! forward. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use super::LEASES; +use super::repo::{Repo, RetryPolicy}; +use crate::document::DocumentStore; +use crate::error::{Result, StorageError}; +use crate::fence::Fence; +use crate::key::Key; +use crate::memory::{Clock, system_clock}; +use crate::scope::Scope; + +/// The stored body: `{holder, epoch, expires_ms}` plus the optional +/// `endpoint` and `released` fields, both absent from records that never +/// used them. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Body { + holder: String, + epoch: u64, + expires_ms: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + endpoint: Option, + #[serde(default, skip_serializing_if = "is_false")] + released: bool, +} + +#[allow(clippy::trivially_copy_pass_by_ref, reason = "serde skip predicate")] +fn is_false(value: &bool) -> bool { + !*value +} + +/// The stored state of one lease, live or not, as [`Leases::inspect`] reads +/// it. [`Lease`] is a holder's proof; this is an observer's view. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseRecord { + /// Who holds (or last held) the lease. + pub holder: String, + /// Where the holder can be reached, when it said so at acquisition. + pub endpoint: Option, + /// The fencing epoch of the holder's tenure. + pub epoch: u64, + /// Unix epoch milliseconds at which the lease lapses. + pub expires_ms: i64, + /// The holder let go cleanly. A released record is never live; a record + /// that is merely past `expires_ms` was abandoned (crash, partition). + pub released: bool, +} + +impl LeaseRecord { + /// Whether the record still excludes other holders at `now_ms`. + #[must_use] + pub fn is_live(&self, now_ms: i64) -> bool { + !self.released && now_ms < self.expires_ms + } + + fn from_body(body: Body) -> Self { + Self { + holder: body.holder, + endpoint: body.endpoint, + epoch: body.epoch, + expires_ms: body.expires_ms, + released: body.released, + } + } +} + +/// What [`Leases::acquire_with`] asks for. +#[derive(Debug, Clone, Copy)] +pub struct Acquire<'a> { + /// The lease namespace. + pub owner: &'a str, + /// The leased key within `owner`. + pub key: &'a str, + /// The acquiring holder. + pub holder: &'a str, + /// How long the lease lasts from now, in milliseconds. + pub ttl_ms: i64, + /// Where the holder can be reached, recorded on the lease. + pub endpoint: Option<&'a str>, + /// The epoch this very holder instance already holds for the key, if + /// any. A live record of the same `holder` string is a re-entrant + /// acquire (same epoch) only when its epoch equals this. With `None` it + /// is a leftover of a dead earlier incarnation and is taken over at once + /// (unclean). With a `Some` that no longer matches, the caller is a + /// stale actor and gets [`Acquisition::Held`]: it must not evict the + /// live replacement. + pub held_epoch: Option, +} + +/// The outcome of [`Leases::acquire_with`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Acquisition { + /// The lease is the caller's. + Granted { + /// The lease just written. + lease: Lease, + /// The acquisition replaced a record nobody released and that the + /// caller did not itself hold: the previous holder's work may be + /// half done. `false` for a fresh key, a released record and a + /// re-entrant acquire. + previous_unclean: bool, + }, + /// Another holder's lease is live. + Held(LeaseRecord), +} + +/// A lease as its holder sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Lease { + id: String, + /// Who holds the lease. + pub holder: String, + /// Moves up each time a different holder takes the lease over, and stays + /// put while one holder renews. This is the write-fence token. + pub epoch: u64, + /// Unix epoch milliseconds at which the lease lapses. + pub expires_ms: i64, + /// Where the holder said it can be reached (see [`Acquire::endpoint`]). + pub endpoint: Option, +} + +impl Lease { + /// A [`Fence`] that holds while this lease's epoch is still the stored + /// one. Bind it with + /// [`StorageBackend::for_scope_fenced`](crate::StorageBackend::for_scope_fenced) + /// so a holder that lost the lease cannot land a late write. + /// + /// `scope` is the scope the lease record lives in, the one the + /// [`Leases`] handle was opened for. + /// + /// The fence checks the epoch only: it refuses writes once another holder + /// took over, not merely because the clock passed `expires_ms`. A holder + /// that wants the second guarantee renews before expiry. + #[must_use] + pub fn fence(&self, scope: &Scope) -> Fence { + Fence::epoch(scope.clone(), LEASES, self.id.clone(), "epoch", self.epoch) + } + + fn from_body(id: String, body: Body) -> Self { + Self { + id, + holder: body.holder, + epoch: body.epoch, + expires_ms: body.expires_ms, + endpoint: body.endpoint, + } + } +} + +/// Leases over [`LEASES`], one record per `(owner, key)`. +/// +/// Time comes from an injectable [`Clock`] (Unix epoch milliseconds), so +/// tests drive expiry without sleeping. A lease is live while +/// `now < expires_ms`. +#[derive(Clone)] +pub struct Leases { + repo: Repo, + clock: Clock, +} + +impl std::fmt::Debug for Leases { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Leases") + .field("repo", &self.repo) + .finish_non_exhaustive() + } +} + +impl Leases { + /// Leases on `docs`, which must be bound to the scope that holds them + /// (typically `cluster` for process placement), timed by the system clock. + /// + /// # Panics + /// + /// Never: the collection name is a valid constant. + #[must_use] + #[allow(clippy::expect_used, reason = "LEASES is a valid constant name")] + pub fn new(docs: &Arc) -> Self { + Self { + repo: Repo::new(docs, LEASES).expect("tsd.leases is a valid collection"), + clock: Arc::new(system_clock), + } + } + + /// Time leases with `clock` instead of the system clock. + #[must_use] + pub fn with_clock(mut self, clock: Clock) -> Self { + self.clock = clock; + self + } + + /// Replace the retry policy of the underlying compare-and-swap loop. + #[must_use] + pub fn with_retry(mut self, retry: RetryPolicy) -> Self { + self.repo = self.repo.with_retry(retry); + self + } + + fn now(&self) -> Result { + i64::try_from((self.clock)()).map_err(|_| StorageError::invalid_input("clock out of range")) + } + + fn deadline(&self, ttl_ms: i64) -> Result<(i64, i64)> { + if ttl_ms <= 0 { + return Err(StorageError::invalid_input("a lease ttl must be positive")); + } + let now = self.now()?; + let expires = now + .checked_add(ttl_ms) + .ok_or_else(|| StorageError::invalid_input("lease ttl out of range"))?; + Ok((now, expires)) + } + + /// Take the lease `(owner, key)` for `holder` for `ttl_ms` milliseconds. + /// + /// Succeeds when the lease is absent (epoch 1), expired or released + /// (epoch + 1, even for the holder that had it, so a fence from the + /// earlier tenure stays dead), or still live and held by `holder` (the + /// epoch stays, the expiry is extended). Returns `None` while another + /// holder's lease is live. Racing acquirers get exactly one winner. + /// + /// # Errors + /// + /// `InvalidInput` for a non-positive ttl, or a storage error. + pub async fn acquire( + &self, + owner: &str, + key: &str, + holder: &str, + ttl_ms: i64, + ) -> Result> { + let id = id(owner, key)?; + let stored = self + .repo + .update(&id, |current| { + let (now, expires_ms) = self.deadline(ttl_ms)?; + let epoch = match current { + None => 1, + Some(lease) if lease.expires_ms > now && !lease.released => { + if lease.holder == holder { + lease.epoch + } else { + return Ok(None); + } + } + Some(lease) => lease + .epoch + .checked_add(1) + .ok_or_else(|| StorageError::backend("lease epoch overflow"))?, + }; + Ok(Some(Body { + holder: holder.to_owned(), + epoch, + expires_ms, + endpoint: None, + released: false, + })) + }) + .await?; + Ok(stored.map(|(body, _)| Lease::from_body(id, body))) + } + + /// Extend a lease `holder` still holds by `ttl_ms` from now, keeping its + /// epoch. `epoch` is the epoch of the [`Lease`] the caller acquired: a + /// holder string is reusable, so an actor from an earlier tenure of the + /// same holder must not be able to renew the current one. Returns `None` + /// when the lease is gone, lapsed, held by someone else or at another + /// epoch: the caller has lost it and must stop acting on it (renewing + /// never resurrects a lapsed lease; call [`acquire`](Self::acquire)). + /// + /// # Errors + /// + /// `InvalidInput` for a non-positive ttl, or a storage error. + pub async fn renew( + &self, + owner: &str, + key: &str, + holder: &str, + epoch: u64, + ttl_ms: i64, + ) -> Result> { + let id = id(owner, key)?; + let stored = self + .repo + .update(&id, |current| { + let (now, expires_ms) = self.deadline(ttl_ms)?; + Ok(current + .filter(|lease| { + lease.holder == holder + && lease.epoch == epoch + && lease.expires_ms > now + && !lease.released + }) + .map(|lease| Body { + expires_ms, + ..lease + })) + }) + .await?; + Ok(stored.map(|(body, _)| Lease::from_body(id, body))) + } + + /// Give the lease up: only the holder of a live lease at `epoch` (the + /// epoch of the [`Lease`] the caller acquired) can, and it is marked + /// expired at once (the record and its epoch stay, so the next holder + /// continues the epoch). Returns whether a lease was released; a stale + /// tenure of the same holder gets `false` and leaves the live one alone. + /// + /// # Errors + /// + /// A storage error. + pub async fn release(&self, owner: &str, key: &str, holder: &str, epoch: u64) -> Result { + let id = id(owner, key)?; + let now = self.now()?; + let stored = self + .repo + .update(&id, |current| { + Ok(current + .filter(|lease| { + lease.holder == holder + && lease.epoch == epoch + && lease.expires_ms > now + && !lease.released + }) + .map(|lease| Body { + expires_ms: now, + released: true, + ..lease + })) + }) + .await?; + Ok(stored.is_some()) + } + + /// The live lease `(owner, key)`, or `None` when absent or lapsed. + /// + /// # Errors + /// + /// A storage error. + pub async fn current(&self, owner: &str, key: &str) -> Result> { + let id = id(owner, key)?; + let now = self.now()?; + Ok(self + .repo + .get(&id) + .await? + .filter(|found| found.doc.expires_ms > now && !found.doc.released) + .map(|found| Lease::from_body(id, found.doc))) + } + + /// Take `(owner, key)` with the full host semantics: an endpoint, a + /// distinction between a clean and an unclean takeover, and an unclean + /// takeover of the same holder's leftover live record. + /// + /// The rule, in order: + /// + /// 1. no record: epoch 1, clean; + /// 2. live, same holder and `held_epoch == Some(record.epoch)`: + /// re-entrant, the epoch stays, clean; + /// 3. released: epoch + 1, clean; + /// 4. live, same holder, `held_epoch == None` (the caller holds nothing: + /// the previous incarnation died before it expired): epoch + 1, + /// unclean, taken over at once, fencing the old incarnation out. A + /// caller that passes a `held_epoch` which no longer matches is a + /// stale actor and gets [`Acquisition::Held`] instead, and so does + /// a call that lost a race to another acquire of the same holder; + /// 5. expired (`now >= expires_ms`): epoch + 1, unclean; + /// 6. otherwise [`Acquisition::Held`] with the live record. + /// + /// Racing acquirers get exactly one winner. + /// + /// # Errors + /// + /// `InvalidInput` for a non-positive ttl, a storage error, or a backend + /// error when the epoch cannot be advanced. + pub async fn acquire_with(&self, request: Acquire<'_>) -> Result { + let Acquire { + owner, + key, + holder, + ttl_ms, + endpoint, + held_epoch, + } = request; + let id = id(owner, key)?; + let mut unclean = false; + let mut seen: Option = None; + // `Repo::update` calls the closure again only after losing the swap, + // so any call after the first means another writer got in: judge the + // record as that competitor's write, never as a dead incarnation. + let mut attempts = 0_u32; + let stored = self + .repo + .update(&id, |current| { + let (now, expires_ms) = self.deadline(ttl_ms)?; + unclean = false; + seen = None; + let raced = attempts > 0; + attempts += 1; + let (epoch, was_unclean) = match ¤t { + None => (1, false), + Some(found) => { + let live = found.expires_ms > now && !found.released; + if live && found.holder == holder && held_epoch == Some(found.epoch) { + (found.epoch, false) + } else if live && (found.holder != holder || raced || held_epoch.is_some()) + { + // Another holder, a competing call of the same + // holder that just won, or a caller proving it + // holds a stale epoch: the live record stands. + seen.clone_from(¤t); + return Ok(None); + } else { + let next = found + .epoch + .checked_add(1) + .ok_or_else(|| StorageError::backend("lease epoch overflow"))?; + (next, !found.released) + } + } + }; + unclean = was_unclean; + Ok(Some(Body { + holder: holder.to_owned(), + epoch, + expires_ms, + endpoint: endpoint.map(str::to_owned), + released: false, + })) + }) + .await?; + Ok(match stored { + Some((body, _)) => Acquisition::Granted { + lease: Lease::from_body(id, body), + previous_unclean: unclean, + }, + None => match seen { + Some(body) => Acquisition::Held(LeaseRecord::from_body(body)), + None => { + return Err(StorageError::backend( + "lease acquire declined without a live record", + )); + } + }, + }) + } + + /// The stored record of `(owner, key)`, live, lapsed or released. Unlike + /// [`current`](Self::current) it does not hide a record that stopped + /// excluding others, so an observer can tell a clean release from an + /// abandoned lease and read the epoch a takeover would continue. + /// + /// # Errors + /// + /// A storage error. + pub async fn inspect(&self, owner: &str, key: &str) -> Result> { + let id = id(owner, key)?; + Ok(self + .repo + .get(&id) + .await? + .map(|found| LeaseRecord::from_body(found.doc))) + } + + /// Drop the record of `(owner, key)` if and only if it still belongs to + /// the tenure `(holder, epoch)`, live or not. Returns whether a record was + /// removed; a record at another holder or epoch (a takeover wrote it) is + /// left alone. + /// + /// This is the compaction primitive for lease keys that are never + /// acquired again (one lease per cron occurrence, per run, per claim), + /// which would otherwise pile up: [`release`](Self::release) deliberately + /// keeps the record so a takeover continues the epoch, and nothing else + /// ever removes it. + /// + /// # Why dropping is safe + /// + /// tsd keeps expired and released leases for one reason: a takeover reads + /// the old record and writes `epoch + 1`, so a stale holder's fence at the + /// old epoch stays dead. The delete is conditional on holder and epoch, so + /// it can only remove the very tenure the caller knows: any takeover bumps + /// the epoch, and a record that moved on is not touched. A [`Fence`] on a + /// removed record refuses every write (a missing fence document is a + /// failed check), so the tenure that was dropped is fenced out for good. + /// + /// What the delete gives up is the epoch history: a later acquire of the + /// same key would restart at epoch 1. The caller therefore must only + /// compact a key that nothing will acquire again (for a cron occurrence: + /// once the job's next run has moved past it), or one whose protected + /// resources no longer hold epoch-1 writers. + /// + /// # Errors + /// + /// A storage error, or a conflict past the retry budget. + pub async fn compact(&self, owner: &str, key: &str, holder: &str, epoch: u64) -> Result { + let id = id(owner, key)?; + self.repo + .delete_if(&id, |lease| lease.holder == holder && lease.epoch == epoch) + .await + } +} + +fn id(owner: &str, key: &str) -> Result { + Ok(Key::of([owner, key])?.as_str().to_owned()) +} + +#[cfg(test)] +#[path = "leases_tests.rs"] +mod tests; diff --git a/crates/tinyflows-storage/src/primitives/repo.rs b/crates/tinyflows-storage/src/primitives/repo.rs new file mode 100644 index 00000000..af9b6d40 --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/repo.rs @@ -0,0 +1,352 @@ +//! [`Repo`]: a typed collection handle and the one compare-and-swap loop. + +use std::collections::hash_map::RandomState; +use std::hash::{BuildHasher, Hasher}; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::document::{ + DocumentStore, Page, Precondition, Query, Version, Versioned, WriteOptions, validate_collection, +}; +use crate::error::{ErrorKind, Result}; + +/// How long [`Repo::update`] keeps retrying a lost compare-and-swap. +/// +/// The default is the spec's budget: 32 attempts with a jittered backoff that +/// starts at 1 ms and doubles up to 8 ms. A lost swap means another writer +/// made progress, so the loop converges; the backoff only spreads the retries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RetryPolicy { + /// Total attempts, including the first. At least one is always made. + pub attempts: u32, + /// The pause before the second attempt; doubles each attempt. + pub initial_backoff: Duration, + /// The longest pause. Each pause is jittered into its upper half. + pub max_backoff: Duration, +} + +impl RetryPolicy { + /// A policy with the given budget and backoff bounds. + #[must_use] + pub const fn new(attempts: u32, initial_backoff: Duration, max_backoff: Duration) -> Self { + Self { + attempts, + initial_backoff, + max_backoff, + } + } + + /// The full budget with no pause between attempts, for tests. + #[must_use] + pub const fn immediate() -> Self { + Self::new(32, Duration::ZERO, Duration::ZERO) + } + + /// The pause after the failed attempt number `attempt` (counting from 0): + /// the doubled backoff capped at the maximum, jittered into its upper half. + fn pause(&self, attempt: u32) -> Duration { + let doubled = self + .initial_backoff + .saturating_mul(1_u32.checked_shl(attempt).unwrap_or(u32::MAX)); + let cap = doubled.min(self.max_backoff); + let nanos = u64::try_from(cap.as_nanos()).unwrap_or(u64::MAX); + if nanos == 0 { + return Duration::ZERO; + } + // `RandomState` draws fresh keys per instance: jitter without a + // random-number dependency. + let random = RandomState::new().build_hasher().finish(); + let half = nanos / 2; + Duration::from_nanos(half + random % (nanos - half + 1)) + } +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self::new(32, Duration::from_millis(1), Duration::from_millis(8)) + } +} + +/// A typed handle on one collection of one scope. +/// +/// Bodies are `T` serialized as a JSON object (see +/// [`validate_doc`](crate::validate_doc)). Every write carries the repo's +/// [`WriteOptions`], so a collection declares its schema version (and, for a +/// short-lived one, an expiry) once. +/// +/// ``` +/// # tokio::runtime::Builder::new_current_thread().build().unwrap().block_on(async { +/// use serde::{Deserialize, Serialize}; +/// use tinystoragedrivers_core::primitives::Repo; +/// use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +/// +/// #[derive(Serialize, Deserialize, PartialEq, Debug)] +/// struct Job { runs: u32 } +/// +/// let storage = MemoryStorage::new().for_scope(&Scope::local())?; +/// let jobs = Repo::::new(storage.documents(), "cron.jobs")?; +/// jobs.update("nightly", |job| Ok(Some(Job { runs: job.map_or(1, |j| j.runs + 1) }))).await?; +/// let (job, _version) = jobs +/// .update("nightly", |job| Ok(Some(Job { runs: job.map_or(1, |j| j.runs + 1) }))) +/// .await? +/// .unwrap(); +/// assert_eq!(job.runs, 2); +/// # Ok::<(), tinystoragedrivers_core::StorageError>(()) +/// # }).unwrap(); +/// ``` +pub struct Repo { + docs: Arc, + collection: String, + options: WriteOptions, + retry: RetryPolicy, + // `fn() -> T` keeps the handle `Send + Sync` whatever `T` is. + marker: PhantomData T>, +} + +impl Clone for Repo { + fn clone(&self) -> Self { + Self { + docs: Arc::clone(&self.docs), + collection: self.collection.clone(), + options: self.options, + retry: self.retry, + marker: PhantomData, + } + } +} + +impl std::fmt::Debug for Repo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Repo") + .field("collection", &self.collection) + .field("options", &self.options) + .field("retry", &self.retry) + .finish_non_exhaustive() + } +} + +impl Repo { + /// A handle on `collection`, with plain writes and the default + /// [`RetryPolicy`]. + /// + /// The collection is not created: declare it with + /// [`ensure_collection`](DocumentStore::ensure_collection) when it needs + /// indexes. + /// + /// # Errors + /// + /// [`ErrorKind::InvalidInput`] for an invalid collection name. + pub fn new(docs: &Arc, collection: impl Into) -> Result { + let collection = collection.into(); + validate_collection(&collection)?; + Ok(Self { + docs: Arc::clone(docs), + collection, + options: WriteOptions::default(), + retry: RetryPolicy::default(), + marker: PhantomData, + }) + } + + /// Declare the schema version and expiry every write of this handle carries. + #[must_use] + pub fn with_options(mut self, options: WriteOptions) -> Self { + self.options = options; + self + } + + /// Replace the [`RetryPolicy`] of [`update`](Self::update). + #[must_use] + pub fn with_retry(mut self, retry: RetryPolicy) -> Self { + self.retry = retry; + self + } + + /// The collection this handle is bound to. + #[must_use] + pub fn collection(&self) -> &str { + &self.collection + } +} + +impl Repo { + /// Read one record. + /// + /// # Errors + /// + /// The read's error, or [`ErrorKind::Serialization`] when the body is not a `T`. + pub async fn get(&self, id: &str) -> Result>> { + match self.docs.get(&self.collection, id).await? { + Some(found) => Ok(Some(decode(found)?)), + None => Ok(None), + } + } + + /// Write a record, replacing any earlier one (upsert). + /// + /// # Errors + /// + /// The write's error, or [`ErrorKind::Serialization`]. + pub async fn put(&self, id: &str, value: &T) -> Result { + self.write(id, value, Precondition::None).await + } + + /// Write a record that must not exist yet. + /// + /// # Errors + /// + /// [`ErrorKind::Conflict`] when the id is taken, or as [`put`](Self::put). + pub async fn insert(&self, id: &str, value: &T) -> Result { + self.write(id, value, Precondition::Absent).await + } + + /// Remove a record and report whether it existed. + /// + /// # Errors + /// + /// The delete's error. + pub async fn delete(&self, id: &str) -> Result { + self.docs + .delete(&self.collection, id, Precondition::None) + .await + } + + /// Remove a record only when `matches` accepts its current value, as one + /// compare-and-swap: the delete is conditional on the version `keep_if` + /// saw, so a writer that got in between turns it into a retry, not a + /// lost update. Returns whether a record was removed; `false` when the + /// record is absent or `matches` declined it. + /// + /// `matches` is called again on every retry, so it must be free of side + /// effects. After [`RetryPolicy::attempts`] lost swaps the last + /// [`Conflict`](ErrorKind::Conflict) is returned. + /// + /// # Errors + /// + /// A conflict past the budget, or any other read or delete error. + pub async fn delete_if(&self, id: &str, mut matches: F) -> Result + where + F: FnMut(&T) -> bool + Send, + { + let attempts = self.retry.attempts.max(1); + let mut attempt = 0; + loop { + let Some(found) = self.docs.get(&self.collection, id).await? else { + return Ok(false); + }; + let found = decode::(found)?; + if !matches(&found.doc) { + return Ok(false); + } + match self + .docs + .delete(&self.collection, id, Precondition::Version(found.version)) + .await + { + Ok(removed) => return Ok(removed), + Err(error) if error.kind() == ErrorKind::Conflict => { + attempt += 1; + if attempt >= attempts { + return Err(error); + } + let pause = self.retry.pause(attempt - 1); + if !pause.is_zero() { + tokio::time::sleep(pause).await; + } + } + Err(error) => return Err(error), + } + } + } + + /// One page of the records matching `query`. + /// + /// # Errors + /// + /// The query's error, or [`ErrorKind::Serialization`]. + pub async fn query(&self, query: &Query) -> Result>> { + let page = self.docs.query(&self.collection, query).await?; + Ok(Page { + items: page + .items + .into_iter() + .map(decode) + .collect::>>()?, + next: page.next, + }) + } + + /// Read-modify-write one record as a single compare-and-swap loop. + /// + /// `f` receives the current value (`None` when the record is missing) and + /// returns the value to store, or `None` to leave the record as it is. + /// The write is conditional on the version `f` saw (or on the record still + /// being absent). When another writer got in first the loop re-reads and + /// calls `f` again, so `f` must be free of side effects. After + /// [`RetryPolicy::attempts`] lost swaps the last + /// [`Conflict`](ErrorKind::Conflict) is returned. + /// + /// Returns the stored value and its version, or `None` when `f` declined. + /// + /// # Errors + /// + /// `f`'s error (nothing is written), a conflict past the budget, or any + /// other read or write error. + pub async fn update(&self, id: &str, mut f: F) -> Result> + where + F: FnMut(Option) -> Result> + Send, + { + let attempts = self.retry.attempts.max(1); + let mut attempt = 0; + loop { + let (precondition, current) = match self.docs.get(&self.collection, id).await? { + Some(found) => { + let found = decode::(found)?; + (Precondition::Version(found.version), Some(found.doc)) + } + None => (Precondition::Absent, None), + }; + let Some(next) = f(current)? else { + return Ok(None); + }; + match self.write(id, &next, precondition).await { + Ok(version) => return Ok(Some((next, version))), + Err(error) if error.kind() == ErrorKind::Conflict => { + attempt += 1; + if attempt >= attempts { + return Err(error); + } + let pause = self.retry.pause(attempt - 1); + if !pause.is_zero() { + tokio::time::sleep(pause).await; + } + } + Err(error) => return Err(error), + } + } + } + + async fn write(&self, id: &str, value: &T, precondition: Precondition) -> Result { + let body = serde_json::to_value(value)?; + self.docs + .put_with(&self.collection, id, body, precondition, self.options) + .await + } +} + +fn decode(found: Versioned) -> Result> { + Ok(Versioned { + id: found.id, + version: found.version, + doc: serde_json::from_value(found.doc)?, + meta: found.meta, + }) +} + +#[cfg(test)] +#[path = "repo_tests.rs"] +mod tests; From a73b2ac7763697329b6cdecdc1c8911cd3132cf4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:12:38 +0300 Subject: [PATCH 03/64] chore: files changed crates/tinyflows-storage/src/lib.rs,crates/tinyflows-storage/Cargo.toml Auto-committed-on: dragonfly --- crates/tinyflows-storage/Cargo.toml | 20 ++++++++++++ crates/tinyflows-storage/src/lib.rs | 48 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 crates/tinyflows-storage/Cargo.toml diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml new file mode 100644 index 00000000..c2804a2e --- /dev/null +++ b/crates/tinyflows-storage/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "tinyflows-storage" +version = "0.1.0" +edition.workspace = true +rust-version = "1.88" +license.workspace = true +repository.workspace = true +publish = false +description = "TinyFlows-owned generic document persistence helpers" + +[dependencies] +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +serde = { workspace = true } +serde_json = { workspace = true } +async-trait = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync"] } + +[dev-dependencies] +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index e69de29b..1b662b99 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -0,0 +1,48 @@ +//! TinyFlows-owned storage helpers built on the public storage ports. +//! +//! These helpers contain record and migration behavior owned by TinyFlows; +//! backend selection stays with the host. + +mod primitives; + +pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; +pub use tinystoragedrivers_types::*; +pub use primitives::{Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy}; + +pub(crate) type Clock = std::sync::Arc u64 + Send + Sync>; + +pub mod primitives { + use std::sync::Arc; + + use tinystoragedrivers_types::{CollectionSpec, IndexSpec, Result}; + + mod counters; + mod kv; + mod leases; + mod repo; + + pub use counters::Counters; + pub use kv::Kv; + pub use leases::{Acquire, Acquisition, Lease, LeaseRecord, Leases}; + pub use repo::{Repo, RetryPolicy}; + + pub const LEASES: &str = "tsd.leases"; + pub const KV: &str = "tsd.kv"; + pub const COUNTERS: &str = "tsd.counters"; + + #[must_use] + pub fn collection_specs() -> Vec { + vec![ + CollectionSpec::new(LEASES), + CollectionSpec::new(KV).index(IndexSpec::new("kv_by_key", ["owner", "ns", "key"])), + CollectionSpec::new(COUNTERS), + ] + } + + pub async fn ensure(docs: &Arc) -> Result<()> { + for spec in collection_specs() { + docs.ensure_collection(&spec).await?; + } + Ok(()) + } +} From 8714967a4948a115d36ab5215d5d6d1926f103fd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:12:56 +0300 Subject: [PATCH 04/64] chore: files changed crates/tinyflows-storage/src/lib.rs,crates/tinyflows-storage/src/primitives/cou Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/lib.rs | 9 +++++++-- crates/tinyflows-storage/src/primitives/counters.rs | 6 +++--- crates/tinyflows-storage/src/primitives/kv.rs | 8 ++++---- crates/tinyflows-storage/src/primitives/leases.rs | 6 +++--- crates/tinyflows-storage/src/primitives/repo.rs | 4 ++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 1b662b99..6dade529 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -3,14 +3,19 @@ //! These helpers contain record and migration behavior owned by TinyFlows; //! backend selection stays with the host. -mod primitives; - pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; pub use tinystoragedrivers_types::*; pub use primitives::{Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy}; pub(crate) type Clock = std::sync::Arc u64 + Send + Sync>; +pub(crate) fn system_clock() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)) +} + pub mod primitives { use std::sync::Arc; diff --git a/crates/tinyflows-storage/src/primitives/counters.rs b/crates/tinyflows-storage/src/primitives/counters.rs index 25dc5d2e..0977cbb1 100644 --- a/crates/tinyflows-storage/src/primitives/counters.rs +++ b/crates/tinyflows-storage/src/primitives/counters.rs @@ -6,9 +6,9 @@ use serde::{Deserialize, Serialize}; use super::COUNTERS; use super::repo::{Repo, RetryPolicy}; -use crate::document::DocumentStore; -use crate::error::{Result, StorageError}; -use crate::key::Key; +use tinystoragedrivers_api::types::DocumentStore; +use tinystoragedrivers_types::{Result, StorageError}; +use tinystoragedrivers_types::Key; /// The largest value every driver can store (`i64::MAX`). const MAX_VALUE: u64 = i64::MAX as u64; diff --git a/crates/tinyflows-storage/src/primitives/kv.rs b/crates/tinyflows-storage/src/primitives/kv.rs index 1f0c64ef..c887e896 100644 --- a/crates/tinyflows-storage/src/primitives/kv.rs +++ b/crates/tinyflows-storage/src/primitives/kv.rs @@ -8,10 +8,10 @@ use serde_json::Value; use super::KV; use super::repo::Repo; -use crate::document::{DocumentStore, Precondition, Query, Version}; -use crate::error::{Result, StorageError}; -use crate::filter::{Filter, Sort}; -use crate::key::Key; +use tinystoragedrivers_api::types::{DocumentStore, Precondition, Query, Version}; +use tinystoragedrivers_types::{Result, StorageError}; +use tinystoragedrivers_types::{Filter, Sort}; +use tinystoragedrivers_types::Key; /// Entries per page while [`Kv::list`] walks a namespace. const PAGE: usize = 200; diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 61a08536..11b411ca 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -17,10 +17,10 @@ use serde::{Deserialize, Serialize}; use super::LEASES; use super::repo::{Repo, RetryPolicy}; -use crate::document::DocumentStore; -use crate::error::{Result, StorageError}; +use tinystoragedrivers_api::types::DocumentStore; +use tinystoragedrivers_types::{Result, StorageError}; use crate::fence::Fence; -use crate::key::Key; +use tinystoragedrivers_types::Key; use crate::memory::{Clock, system_clock}; use crate::scope::Scope; diff --git a/crates/tinyflows-storage/src/primitives/repo.rs b/crates/tinyflows-storage/src/primitives/repo.rs index af9b6d40..a9164b1e 100644 --- a/crates/tinyflows-storage/src/primitives/repo.rs +++ b/crates/tinyflows-storage/src/primitives/repo.rs @@ -9,10 +9,10 @@ use std::time::Duration; use serde::Serialize; use serde::de::DeserializeOwned; -use crate::document::{ +use tinystoragedrivers_api::types::{ DocumentStore, Page, Precondition, Query, Version, Versioned, WriteOptions, validate_collection, }; -use crate::error::{ErrorKind, Result}; +use tinystoragedrivers_types::{ErrorKind, Result}; /// How long [`Repo::update`] keeps retrying a lost compare-and-swap. /// From 329d874a24c9829bdf5651a0519dd754d0dde97f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:13:12 +0300 Subject: [PATCH 05/64] refactor(leases): remove fence implementation from lease The epoch-based fence functionality has been removed from the Lease struct. This simplifies the lease management by removing the dependency on the Fence mechanism for epoch checking. Auto-committed-on: dragonfly --- .../src/primitives/leases.rs | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 11b411ca..5104d15d 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -17,12 +17,10 @@ use serde::{Deserialize, Serialize}; use super::LEASES; use super::repo::{Repo, RetryPolicy}; -use tinystoragedrivers_api::types::DocumentStore; +use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; -use crate::fence::Fence; use tinystoragedrivers_types::Key; -use crate::memory::{Clock, system_clock}; -use crate::scope::Scope; +use crate::{Clock, system_clock}; /// The stored body: `{holder, epoch, expires_ms}` plus the optional /// `endpoint` and `released` fields, both absent from records that never @@ -134,22 +132,6 @@ pub struct Lease { } impl Lease { - /// A [`Fence`] that holds while this lease's epoch is still the stored - /// one. Bind it with - /// [`StorageBackend::for_scope_fenced`](crate::StorageBackend::for_scope_fenced) - /// so a holder that lost the lease cannot land a late write. - /// - /// `scope` is the scope the lease record lives in, the one the - /// [`Leases`] handle was opened for. - /// - /// The fence checks the epoch only: it refuses writes once another holder - /// took over, not merely because the clock passed `expires_ms`. A holder - /// that wants the second guarantee renews before expiry. - #[must_use] - pub fn fence(&self, scope: &Scope) -> Fence { - Fence::epoch(scope.clone(), LEASES, self.id.clone(), "epoch", self.epoch) - } - fn from_body(id: String, body: Body) -> Self { Self { id, From c5b3d6c0f76816938f6608d53d146b72777391d0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:13:24 +0300 Subject: [PATCH 06/64] refactor(storage): update imports in primitives This change updates the import paths for types used within the storage primitives. This refactoring standardizes how types from `tinystoragedrivers_api` are brought into scope across the repository and counter modules. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/primitives/counters.rs | 2 +- crates/tinyflows-storage/src/primitives/repo.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyflows-storage/src/primitives/counters.rs b/crates/tinyflows-storage/src/primitives/counters.rs index 0977cbb1..7acba097 100644 --- a/crates/tinyflows-storage/src/primitives/counters.rs +++ b/crates/tinyflows-storage/src/primitives/counters.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use super::COUNTERS; use super::repo::{Repo, RetryPolicy}; -use tinystoragedrivers_api::types::DocumentStore; +use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; use tinystoragedrivers_types::Key; diff --git a/crates/tinyflows-storage/src/primitives/repo.rs b/crates/tinyflows-storage/src/primitives/repo.rs index a9164b1e..1fef0157 100644 --- a/crates/tinyflows-storage/src/primitives/repo.rs +++ b/crates/tinyflows-storage/src/primitives/repo.rs @@ -9,7 +9,7 @@ use std::time::Duration; use serde::Serialize; use serde::de::DeserializeOwned; -use tinystoragedrivers_api::types::{ +use tinystoragedrivers_api::{ DocumentStore, Page, Precondition, Query, Version, Versioned, WriteOptions, validate_collection, }; use tinystoragedrivers_types::{ErrorKind, Result}; From e0624c42495c22e4450a034933e552d78d781c74 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:14:35 +0300 Subject: [PATCH 07/64] chore(deps): update dependencies and add new driver packages This update adds several new driver packages to the workspace and updates dependencies across the project. It also exposes the `Key` type from the primitives module in `lib.rs`. Auto-committed-on: dragonfly --- Cargo.lock | 151 ++++++++++++ crates/tinyflows-storage/Cargo.toml | 3 + crates/tinyflows-storage/src/key.rs | 217 ++++++++++++++++++ crates/tinyflows-storage/src/lib.rs | 2 + crates/tinyflows-storage/src/primitives/kv.rs | 6 +- .../tinyflows-storage/src/primitives/repo.rs | 7 +- 6 files changed, 379 insertions(+), 7 deletions(-) create mode 100644 crates/tinyflows-storage/src/key.rs diff --git a/Cargo.lock b/Cargo.lock index 25f97e01..d596251b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -539,6 +539,17 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -564,6 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", + "futures-macro", "futures-sink", "futures-task", "pin-project-lite", @@ -2216,6 +2228,57 @@ dependencies = [ "uuid", ] +[[package]] +name = "tinyflows-storage" +version = "0.1.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "tinystoragedrivers-api", + "tinystoragedrivers-testkit", + "tinystoragedrivers-types", + "tokio", +] + +[[package]] +name = "tinystoragedrivers-api" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "bytes", + "futures-core", + "serde", + "serde_json", + "tinystoragedrivers-types", + "zeroize", +] + +[[package]] +name = "tinystoragedrivers-blob" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "bytes", + "futures-core", + "serde", + "serde_json", + "tinystoragedrivers-api", + "tinystoragedrivers-types", +] + +[[package]] +name = "tinystoragedrivers-connect" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "rusqlite", + "tinystoragedrivers-types", + "tokio", +] + [[package]] name = "tinystoragedrivers-core" version = "1.0.0" @@ -2232,6 +2295,63 @@ dependencies = [ "zstd", ] +[[package]] +name = "tinystoragedrivers-document" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "data-encoding", + "rusqlite", + "serde", + "serde_json", + "sha2 0.10.9", + "tinystoragedrivers-api", + "tinystoragedrivers-connect", + "tinystoragedrivers-types", + "tokio", + "zstd", +] + +[[package]] +name = "tinystoragedrivers-kv" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "tinystoragedrivers-api", + "tinystoragedrivers-types", +] + +[[package]] +name = "tinystoragedrivers-queue" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "data-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "tinystoragedrivers-api", + "tinystoragedrivers-types", + "tokio", + "ulid", +] + +[[package]] +name = "tinystoragedrivers-secrets" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "tinystoragedrivers-api", + "tinystoragedrivers-types", + "zeroize", +] + [[package]] name = "tinystoragedrivers-sqlite" version = "1.0.0" @@ -2245,6 +2365,37 @@ dependencies = [ "zstd", ] +[[package]] +name = "tinystoragedrivers-testkit" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "serde_json", + "tinystoragedrivers-api", + "tinystoragedrivers-blob", + "tinystoragedrivers-document", + "tinystoragedrivers-kv", + "tinystoragedrivers-queue", + "tinystoragedrivers-secrets", + "tinystoragedrivers-types", + "tokio", + "ulid", + "zeroize", +] + +[[package]] +name = "tinystoragedrivers-types" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "tinystr" version = "0.8.4" diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index c2804a2e..5c649710 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -15,6 +15,9 @@ serde = { workspace = true } serde_json = { workspace = true } async-trait = { workspace = true } tokio = { workspace = true, features = ["rt", "sync"] } +sha2 = "0.11" +ulid = "1" +data-encoding = "2" [dev-dependencies] tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } diff --git a/crates/tinyflows-storage/src/key.rs b/crates/tinyflows-storage/src/key.rs new file mode 100644 index 00000000..1a41a190 --- /dev/null +++ b/crates/tinyflows-storage/src/key.rs @@ -0,0 +1,217 @@ +//! Document keys: composite ids built from parts, and entity ids from ULIDs. +//! +//! A [`Key`] is a document id that already satisfies the id rules +//! ([`validate_id`]): 1 to [`MAX_ID_LEN`] bytes, no NUL. Two constructors cover +//! the two ways a record gets its id: +//! +//! - [`Key::of`] addresses a record by what it is (`job` / `7f` / a timestamp) +//! and is reversible by [`Key::split`] until the key outgrows the limit. +//! - [`Key::ulid`] names a new entity with a 26-character id that sorts by +//! creation time. +//! +//! # Composite form +//! +//! Each part is escaped, then the parts are joined with `/`: +//! +//! | Byte | Written as | +//! | --- | --- | +//! | `%` | `%25` | +//! | `/` | `%2F` | +//! | NUL | `%00` | +//! | a leading `~` in the first part | `%7E` | +//! +//! The last row is what keeps the hashed form unambiguous (below): an ordinary +//! key never starts with `~`. +//! +//! If the escaped key is longer than [`MAX_ID_LEN`] bytes it is replaced by `~` +//! followed by the lowercase RFC 4648 base32 (no padding) of its SHA-256 +//! digest, the full 52 characters. An id is a primary key, so the digest is not +//! truncated: 256 bits keep a collision out of reach, where the 130 bits a +//! shortened name would keep are a choice only a segment label can afford. The +//! digest covers the *escaped* key, so two part lists that join to the same +//! escaped text hash alike and none can alias another. +//! +//! ``` +//! use tinystoragedrivers_core::Key; +//! +//! let key = Key::of(["job", "a/b"])?; +//! assert_eq!(key.as_str(), "job/a%2Fb"); +//! assert_eq!(key.split().unwrap(), ["job", "a/b"]); +//! +//! let long = Key::of(["x".repeat(600)])?; +//! assert!(long.is_hashed()); +//! assert_eq!(long.split(), None); +//! # Ok::<(), tinystoragedrivers_core::StorageError>(()) +//! ``` + +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use sha2::{Digest, Sha256}; +use ulid::Ulid; + +use tinystoragedrivers_types::{MAX_ID_LEN, validate_id}; +use tinystoragedrivers_types::{Result, StorageError}; + +/// Marks a key that was replaced by its digest. +const HASH_MARK: char = '~'; + +/// A validated document id. +/// +/// Serializes as a plain JSON string; deserializing applies the id rules, so a +/// stored id that no longer validates is rejected instead of passed along. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Key(String); + +impl Key { + /// Join `parts` into one escaped key, hashing it if it exceeds + /// [`MAX_ID_LEN`] bytes. + /// + /// Empty parts are allowed and encode as nothing, so `["a", "", "b"]` + /// becomes `a//b`. + /// + /// # Errors + /// + /// [`ErrorKind::InvalidInput`](crate::ErrorKind::InvalidInput) when `parts` + /// is empty, or holds only one empty part (an empty id is not a valid id). + pub fn of(parts: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut escaped = String::new(); + let mut count = 0usize; + for part in parts { + if count > 0 { + escaped.push('/'); + } + escape_into(part.as_ref(), count == 0, &mut escaped); + count += 1; + } + if count == 0 { + return Err(StorageError::invalid_input("a key needs at least one part")); + } + if escaped.len() > MAX_ID_LEN { + escaped = format!("{HASH_MARK}{}", base32_lower(&Sha256::digest(&escaped))); + } + Self::new(escaped) + } + + /// Wrap an id that is already in key form, such as one read back from + /// storage or a hand-written constant. + /// + /// # Errors + /// + /// [`ErrorKind::InvalidInput`](crate::ErrorKind::InvalidInput) under the + /// rules of [`validate_id`]: empty, longer than [`MAX_ID_LEN`] bytes, or + /// holding NUL. + pub fn new(raw: impl Into) -> Result { + let raw = raw.into(); + validate_id(&raw)?; + Ok(Self(raw)) + } + + /// A new ULID: 26 Crockford base32 characters whose lexical order is + /// creation order, to the millisecond. Ids made within one millisecond + /// order randomly. + #[must_use] + pub fn ulid() -> Self { + Self(Ulid::new().to_string()) + } + + /// A ULID stamped with `timestamp_ms` (Unix epoch milliseconds) and a + /// random tail, for a migration that keeps a record's original creation + /// time in its new id. + #[must_use] + pub fn ulid_at(timestamp_ms: u64) -> Self { + Self(Ulid::from_parts(timestamp_ms, Ulid::new().random()).to_string()) + } + + /// The parts this key was built from, unescaped. + /// + /// `None` for a hashed key (the parts are not recoverable) and for text + /// that is not valid escaped output: a `%` that does not start `%25`, + /// `%2F`, `%00` or `%7E`. + #[must_use] + pub fn split(&self) -> Option> { + if self.is_hashed() { + return None; + } + self.0.split('/').map(unescape).collect() + } + + /// Whether this key is the digest of a longer one. + #[must_use] + pub fn is_hashed(&self) -> bool { + self.0.starts_with(HASH_MARK) + } + + /// The key as text, the form stored as the document id. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +fn escape_into(part: &str, first: bool, out: &mut String) { + for (i, c) in part.chars().enumerate() { + match c { + '%' => out.push_str("%25"), + '/' => out.push_str("%2F"), + '\0' => out.push_str("%00"), + HASH_MARK if first && i == 0 => out.push_str("%7E"), + c => out.push(c), + } + } +} + +fn unescape(segment: &str) -> Option { + let mut out = String::with_capacity(segment.len()); + let mut rest = segment; + while let Some(at) = rest.find('%') { + out.push_str(&rest[..at]); + let tail = &rest[at..]; + let (decoded, len) = [("%25", '%'), ("%2F", '/'), ("%00", '\0'), ("%7E", '~')] + .into_iter() + .find_map(|(escape, c)| tail.starts_with(escape).then_some((c, escape.len())))?; + out.push(decoded); + rest = &tail[len..]; + } + out.push_str(rest); + Some(out) +} + +/// RFC 4648 base32, lowercase, without padding. +fn base32_lower(bytes: &[u8]) -> String { + data_encoding::BASE32_NOPAD + .encode(bytes) + .to_ascii_lowercase() +} + +impl fmt::Display for Key { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl AsRef for Key { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl Serialize for Key { + fn serialize(&self, serializer: S) -> std::result::Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for Key { + fn deserialize>(deserializer: D) -> std::result::Result { + Self::new(String::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 6dade529..a274b7fc 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -5,6 +5,8 @@ pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; pub use tinystoragedrivers_types::*; +mod key; +pub use key::Key; pub use primitives::{Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy}; pub(crate) type Clock = std::sync::Arc u64 + Send + Sync>; diff --git a/crates/tinyflows-storage/src/primitives/kv.rs b/crates/tinyflows-storage/src/primitives/kv.rs index c887e896..b26fdda5 100644 --- a/crates/tinyflows-storage/src/primitives/kv.rs +++ b/crates/tinyflows-storage/src/primitives/kv.rs @@ -8,10 +8,8 @@ use serde_json::Value; use super::KV; use super::repo::Repo; -use tinystoragedrivers_api::types::{DocumentStore, Precondition, Query, Version}; -use tinystoragedrivers_types::{Result, StorageError}; -use tinystoragedrivers_types::{Filter, Sort}; -use tinystoragedrivers_types::Key; +use tinystoragedrivers_api::DocumentStore; +use tinystoragedrivers_types::{Filter, Key, Precondition, Query, Result, Sort, StorageError, Version}; /// Entries per page while [`Kv::list`] walks a namespace. const PAGE: usize = 200; diff --git a/crates/tinyflows-storage/src/primitives/repo.rs b/crates/tinyflows-storage/src/primitives/repo.rs index 1fef0157..90ba83a3 100644 --- a/crates/tinyflows-storage/src/primitives/repo.rs +++ b/crates/tinyflows-storage/src/primitives/repo.rs @@ -9,10 +9,11 @@ use std::time::Duration; use serde::Serialize; use serde::de::DeserializeOwned; -use tinystoragedrivers_api::{ - DocumentStore, Page, Precondition, Query, Version, Versioned, WriteOptions, validate_collection, +use tinystoragedrivers_api::DocumentStore; +use tinystoragedrivers_types::{ + ErrorKind, Page, Precondition, Query, Result, Version, Versioned, WriteOptions, + validate_collection, }; -use tinystoragedrivers_types::{ErrorKind, Result}; /// How long [`Repo::update`] keeps retrying a lost compare-and-swap. /// From 4f16b9b2ae825a36aa7656632a10f6df0dc95cb3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:15:52 +0300 Subject: [PATCH 08/64] chore: files changed Cargo.lock,crates/tinyflows-adaptive/Cargo.toml,crates/tinyflows-adaptive/src/d Auto-committed-on: dragonfly --- Cargo.lock | 3 + crates/tinyflows-adaptive/Cargo.toml | 10 ++- .../src/drivers/ledger/mod.rs | 4 +- .../src/drivers/ledger/trait_impl.rs | 4 +- .../src/drivers/ledger_tests.rs | 4 +- .../tinyflows-adaptive/src/drivers/migrate.rs | 6 +- .../src/drivers/migrate_tests.rs | 4 +- crates/tinyflows-adaptive/src/drivers/mod.rs | 6 +- .../tinyflows-adaptive/src/drivers/vault.rs | 2 +- .../src/drivers/vault_tests.rs | 4 +- crates/tinyflows-adaptive/src/storage.rs | 2 +- .../tinyflows-adaptive/src/storage_tests.rs | 2 +- crates/tinyflows-drivers/Cargo.toml | 9 ++- .../src/catalog/definitions.rs | 2 +- .../src/catalog/definitions_tests.rs | 4 +- .../tinyflows-drivers/src/catalog/drafts.rs | 2 +- .../src/catalog/drafts_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/import.rs | 2 +- crates/tinyflows-drivers/src/catalog/kv.rs | 2 +- .../tinyflows-drivers/src/catalog/kv_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/lineage.rs | 2 +- .../src/catalog/lineage_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/migrate.rs | 2 +- .../src/catalog/migrate_tests.rs | 2 +- crates/tinyflows-drivers/src/catalog/mod.rs | 6 +- .../src/catalog/revisions.rs | 2 +- .../src/catalog/revisions_gap_tests.rs | 2 +- crates/tinyflows-drivers/src/catalog/runs.rs | 2 +- .../src/catalog/runs_lifecycle_tests.rs | 2 +- .../src/catalog/runs_recovery.rs | 2 +- .../src/catalog/runs_recovery_lease_tests.rs | 6 +- .../src/catalog/runs_replace_tests.rs | 6 +- crates/tinyflows-drivers/src/catalog/state.rs | 2 +- .../src/catalog/state_tests.rs | 4 +- .../src/catalog/suggestions.rs | 2 +- .../src/catalog/test_support.rs | 52 ++++++------- crates/tinyflows-drivers/src/checkpoint.rs | 2 +- .../src/checkpoint_import_tests.rs | 4 +- .../src/checkpoint_layout.rs | 2 +- .../src/checkpoint_layout_tests.rs | 6 +- .../tinyflows-drivers/src/checkpoint_tests.rs | 2 +- crates/tinyflows-drivers/src/ids.rs | 2 +- .../src/schedule/admission.rs | 10 +-- .../src/schedule/admission_tests.rs | 6 +- .../src/schedule/claim_removal_tests.rs | 2 +- .../tinyflows-drivers/src/schedule/claims.rs | 2 +- .../src/schedule/claims/acquisition.rs | 2 +- .../src/schedule/claims/completion.rs | 2 +- .../src/schedule/claims_compaction_tests.rs | 2 +- .../src/schedule/claims_safety_tests.rs | 2 +- .../src/schedule/claims_tests.rs | 10 +-- .../tinyflows-drivers/src/schedule/codec.rs | 2 +- .../src/schedule/codec_tests.rs | 2 +- .../tinyflows-drivers/src/schedule/import.rs | 4 +- .../src/schedule/import_tests.rs | 2 +- crates/tinyflows-drivers/src/schedule/jobs.rs | 2 +- .../src/schedule/manual_claim_fence_tests.rs | 24 +++--- .../src/schedule/manual_claim_tests.rs | 2 +- .../tinyflows-drivers/src/schedule/migrate.rs | 4 +- .../src/schedule/migrate_tests.rs | 2 +- crates/tinyflows-drivers/src/schedule/mod.rs | 4 +- .../src/schedule/mod_tests.rs | 4 +- .../tinyflows-drivers/src/schedule/policy.rs | 2 +- .../tinyflows-drivers/src/schedule/removal.rs | 6 +- .../src/schedule/removal_claims_tests.rs | 6 +- crates/tinyflows-drivers/src/schedule/runs.rs | 4 +- .../src/schedule/runs_sweep_tests.rs | 2 +- .../tinyflows-drivers/src/schedule/runtime.rs | 2 +- .../src/schedule/test_support.rs | 66 ++++++++--------- crates/tinyflows-drivers/src/state_store.rs | 2 +- .../src/state_store_tests.rs | 2 +- .../src/storage/kv_import.rs | 2 +- .../src/storage/kv_import_tests.rs | 2 +- crates/tinyflows-drivers/src/storage/mod.rs | 4 +- .../src/store/authoring_tests.rs | 2 +- .../src/store/concurrency_tests.rs | 2 +- .../src/store/definitions.rs | 2 +- crates/tinyflows-drivers/src/store/history.rs | 2 +- .../src/store/history_tests.rs | 2 +- crates/tinyflows-drivers/src/store/import.rs | 2 +- .../src/store/import_tests.rs | 2 +- crates/tinyflows-drivers/src/store/locks.rs | 2 +- .../src/store/locks_tests.rs | 2 +- crates/tinyflows-drivers/src/store/migrate.rs | 2 +- .../src/store/migrate_tests.rs | 4 +- crates/tinyflows-drivers/src/store/mod.rs | 2 +- .../tinyflows-drivers/src/store/mod_tests.rs | 2 +- crates/tinyflows-drivers/src/store/notes.rs | 2 +- .../src/store/notes_tests.rs | 2 +- .../tinyflows-drivers/src/store/proposals.rs | 2 +- .../src/store/proposals_tests.rs | 4 +- crates/tinyflows-drivers/src/store/runs.rs | 2 +- .../tinyflows-drivers/src/store/runs_tests.rs | 2 +- .../src/store/test_support.rs | 74 +++++++++---------- crates/tinyflows-module/Cargo.toml | 7 +- crates/tinyflows-module/src/caps.rs | 2 +- .../src/caps_tests/scope_tests.rs | 4 +- .../tinyflows-module/src/catalog_documents.rs | 4 +- .../src/catalog_extended_operations_tests.rs | 2 +- .../src/catalog_extended_ops.rs | 2 +- .../src/catalog_legacy_import.rs | 2 +- .../src/catalog_migrations.rs | 2 +- .../tinyflows-module/src/catalog_service.rs | 2 +- .../src/catalog_service_tests.rs | 8 +- .../src/run_manager/cancellation.rs | 2 +- .../src/run_manager/dedup_settlement_tests.rs | 2 +- .../events_persistence_test_support.rs | 24 +++--- .../src/run_manager/preparation.rs | 2 +- .../src/run_manager/resume.rs | 2 +- .../src/run_manager/sweeps.rs | 2 +- .../src/run_manager_actor_recovery_tests.rs | 2 +- .../tinyflows-module/src/run_manager_tests.rs | 2 +- .../src/schedule_clock_tests.rs | 4 +- .../src/schedule_documents.rs | 4 +- .../src/schedule_documents_tests.rs | 2 +- ...e_manual_worker_lifecycle_service_tests.rs | 4 +- .../schedule_manual_worker_service_tests.rs | 8 +- .../src/schedule_members_tests.rs | 6 +- .../src/schedule_service_tests.rs | 12 +-- crates/tinyflows-module/src/schedule_store.rs | 2 +- .../src/schedule_store_tests.rs | 2 +- crates/tinyflows-storage/src/lib.rs | 2 +- .../src/primitives/counters.rs | 2 +- crates/tinyflows-storage/src/primitives/kv.rs | 3 +- .../src/primitives/leases.rs | 2 +- 125 files changed, 318 insertions(+), 304 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d596251b..4b4017b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2233,12 +2233,15 @@ name = "tinyflows-storage" version = "0.1.0" dependencies = [ "async-trait", + "data-encoding", "serde", "serde_json", + "sha2 0.11.0", "tinystoragedrivers-api", "tinystoragedrivers-testkit", "tinystoragedrivers-types", "tokio", + "ulid", ] [[package]] diff --git a/crates/tinyflows-adaptive/Cargo.toml b/crates/tinyflows-adaptive/Cargo.toml index 7a7c0d37..0b801ded 100644 --- a/crates/tinyflows-adaptive/Cargo.toml +++ b/crates/tinyflows-adaptive/Cargo.toml @@ -19,7 +19,9 @@ thiserror = "2" # tag so a host that vendors tinystoragedrivers once can `[patch]` it; chrono # turns the RFC 3339 instants of the records into the stored `*_ms`; tokio's # `OnceCell` for one-time collection setup. -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7", optional = true } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1", optional = true } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1", optional = true } +tinyflows-storage = { path = "../tinyflows-storage", optional = true } chrono = { version = "0.4", default-features = false, features = ["std"], optional = true } tracing = { version = "0.1", default-features = false, features = ["std"], optional = true } tokio = { version = "1", features = ["sync"], optional = true } @@ -32,14 +34,16 @@ default = [] # The ledger and vault on a `tinystoragedrivers` document port # (`Storage::from_documents`), so the host's storage backend (SQLite, MongoDB, # memory) holds them alongside everything else. Needs Rust 1.88. -storage-drivers = ["dep:tinystoragedrivers-core", "dep:chrono", "dep:tokio", "dep:tracing"] +storage-drivers = ["dep:tinystoragedrivers-api", "dep:tinystoragedrivers-types", "dep:tinyflows-storage", "dep:chrono", "dep:tokio", "dep:tracing"] [dev-dependencies] tinyflows = { path = "../tinyflows", features = ["store", "mock"] } # The engine's `WorkflowStore` on an in-memory storage backend, for tests and # the eval example. tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["store"] } -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [lints.rust] diff --git a/crates/tinyflows-adaptive/src/drivers/ledger/mod.rs b/crates/tinyflows-adaptive/src/drivers/ledger/mod.rs index a90ce21d..9542e059 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger/mod.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger/mod.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use serde_json::{Map, Value, json}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Counters, DocumentStore, DocumentStoreExt, Filter, Precondition, Query, StorageError, Versioned, WriteOptions, primitives, }; @@ -54,7 +54,7 @@ pub(super) async fn put_doc( id: &str, doc: Value, precondition: Precondition, -) -> std::result::Result { +) -> std::result::Result { docs.put_with( collection, id, diff --git a/crates/tinyflows-adaptive/src/drivers/ledger/trait_impl.rs b/crates/tinyflows-adaptive/src/drivers/ledger/trait_impl.rs index 0e75b391..00cfc71b 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger/trait_impl.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger/trait_impl.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use serde_json::{Value, json}; -use tinystoragedrivers_core::{Filter, Precondition, Query, Sort}; +use tinyflows_storage::{Filter, Precondition, Query, Sort}; /// A document id for `parts`, as a ledger error. fn key(parts: &[&str]) -> Result { @@ -106,7 +106,7 @@ impl Ledger for DriverLedger { } let query = Query::filter( self.visible() - .and(Filter::one_of(tinystoragedrivers_core::ID_FIELD, cited)), + .and(Filter::one_of(tinyflows_storage::ID_FIELD, cited)), ) .sort(Sort::asc("seq")); self.all(ROWS, query) diff --git a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs index f5d1c3e1..e3d79ede 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs @@ -2,8 +2,8 @@ use super::*; use crate::drivers::key; use crate::execute::StepRecord; use crate::ledger::{Ledger, Score, conformance}; -use tinystoragedrivers_core::Counters; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::Counters; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index 00a14ccc..82c8200f 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -20,7 +20,7 @@ //! visible (readers report it) instead of being lost. use serde_json::{Map, Value}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Cursor, Filter, ID_FIELD, Key, Migrations, Query, Result, Schema, ScopeClass, ScopedStorage, Sort, SourceRecord, Step, TargetRecord, primitives, }; @@ -231,9 +231,9 @@ pub async fn import_legacy_adaptive_counters(storage: &ScopedStorage) -> Result< } primitives::ensure(storage).await?; let docs = storage.documents(); - docs.ensure_collection(&tinystoragedrivers_core::CollectionSpec::new(COUNTERS)) + docs.ensure_collection(&tinyflows_storage::CollectionSpec::new(COUNTERS)) .await?; - let counters = tinystoragedrivers_core::Counters::new(docs); + let counters = tinyflows_storage::Counters::new(docs); let mut carried = 0_u64; let mut cursor: Option = None; loop { diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index 2464c131..d625ffd5 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use serde_json::{Value, json}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Counters, DocumentStore, DocumentStoreExt, Key, MemoryStorage, MigrateOptions, Owner, Precondition, Registry, Schema, Scope, ScopeClass, ScopedStorage, StorageBackend, }; @@ -222,7 +222,7 @@ async fn migrating_twice_changes_nothing() { let all = docs(&storage) .query_all( "flows.adaptive_variants", - &tinystoragedrivers_core::Query::all(), + &tinyflows_storage::Query::all(), ) .await .unwrap(); diff --git a/crates/tinyflows-adaptive/src/drivers/mod.rs b/crates/tinyflows-adaptive/src/drivers/mod.rs index abb4c56b..060604e1 100644 --- a/crates/tinyflows-adaptive/src/drivers/mod.rs +++ b/crates/tinyflows-adaptive/src/drivers/mod.rs @@ -48,7 +48,7 @@ mod ledger; mod migrate; mod vault; -use tinystoragedrivers_core::{CollectionSpec, IndexSpec, StorageError}; +use tinyflows_storage::{CollectionSpec, IndexSpec, StorageError}; pub use ledger::DriverLedger; pub use migrate::{extend_storage_schema, import_legacy_adaptive_counters}; @@ -96,12 +96,12 @@ pub(crate) fn collections() -> Vec { /// A document id for `parts` (`Key::of`: escaped so no two tuples collide, /// hashed when it would exceed the driver's id limit). fn key(parts: &[&str]) -> Result { - tinystoragedrivers_core::Key::of(parts.iter().copied()).map(|key| key.as_str().to_owned()) + tinyflows_storage::Key::of(parts.iter().copied()).map(|key| key.as_str().to_owned()) } /// Whether `error` means another writer got there first. fn is_race(error: &StorageError) -> bool { - error.kind() == tinystoragedrivers_core::ErrorKind::Conflict + error.kind() == tinyflows_storage::ErrorKind::Conflict } #[cfg(test)] diff --git a/crates/tinyflows-adaptive/src/drivers/vault.rs b/crates/tinyflows-adaptive/src/drivers/vault.rs index 1f551ea9..ca77541a 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use async_trait::async_trait; use tinyflows::store::types::{WorkflowError, WorkflowRecord}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStore, DocumentStoreExt, Filter, Precondition, Query, Sort, StorageError, WriteOptions, primitives, }; diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index 2f9cce7d..b9e4356d 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -1,8 +1,8 @@ use super::*; use crate::workflows::conformance; use serde_json::json; -use tinystoragedrivers_core::DocumentStoreExt; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::DocumentStoreExt; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/storage.rs b/crates/tinyflows-adaptive/src/storage.rs index 615b8b0b..174bbf2f 100644 --- a/crates/tinyflows-adaptive/src/storage.rs +++ b/crates/tinyflows-adaptive/src/storage.rs @@ -182,7 +182,7 @@ impl Storage { #[cfg(feature = "storage-drivers")] #[must_use] pub fn from_documents( - docs: std::sync::Arc, + docs: std::sync::Arc, ) -> Self { Self { ledger: AnyLedger::Driver(crate::drivers::DriverLedger::new(std::sync::Arc::clone( diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index ae7f8c33..54b6fcf8 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -64,7 +64,7 @@ async fn one_call_scopes_both_halves() { #[cfg(feature = "storage-drivers")] #[tokio::test] async fn one_document_handle_holds_both_halves_and_survives_reopening() { - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let scoped = MemoryStorage::new() .for_scope(&Scope::local()) .expect("scope"); diff --git a/crates/tinyflows-drivers/Cargo.toml b/crates/tinyflows-drivers/Cargo.toml index 2f8c6571..33aeb642 100644 --- a/crates/tinyflows-drivers/Cargo.toml +++ b/crates/tinyflows-drivers/Cargo.toml @@ -31,7 +31,7 @@ catalog = [ "dep:anyhow", "dep:chrono", "dep:uuid", - "tinystoragedrivers-core/blocking", + "tinyflows-storage", ] # `schedule`: the cron job/run store (`schedule::CronDocuments`). It needs only # `tinyflows-schedule`, so a host that runs scheduled jobs but no flows can @@ -45,7 +45,7 @@ store = [ "engine", "tinyflows/store", "dep:uuid", - "tinystoragedrivers-core/blocking", + "tinyflows-storage", ] # The fixture-only writers (`force_run_status_for_test`, # `force_corrupt_graph_json_for_test`) that stage states no production path can @@ -53,7 +53,9 @@ store = [ test-fixtures = ["catalog"] [dependencies] -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinyflows-storage = { path = "../tinyflows-storage" } serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } @@ -72,3 +74,4 @@ uuid = { workspace = true, optional = true } [dev-dependencies] async-trait = { workspace = true } tokio = { workspace = true } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } diff --git a/crates/tinyflows-drivers/src/catalog/definitions.rs b/crates/tinyflows-drivers/src/catalog/definitions.rs index 003556dd..75648d80 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions.rs @@ -6,7 +6,7 @@ use chrono::Utc; use serde_json::{Value, json}; use tinyflows::model::WorkflowGraph; use tinyflows_catalog::Flow; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Sort, Versioned, }; use uuid::Uuid; diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index f6a00f7d..d63d5911 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::catalog::test_support::{catalog, catalog_in, trigger_graph}; use std::sync::Arc; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn create_get_list_delete_roundtrip() { @@ -130,7 +130,7 @@ async fn settled_run_updates_its_workflow_summary_only_for_the_winning_status() #[tokio::test] async fn a_run_from_a_removed_flow_incarnation_cannot_stamp_its_replacement() { use crate::catalog::{DEFINITIONS, RUNS, test_support::docs_in}; - use tinystoragedrivers_core::{MemoryStorage, Precondition}; + use tinyflows_storage::{MemoryStorage, Precondition}; let storage = MemoryStorage::new(); let store = catalog_in(&storage, "incarnation-summary"); diff --git a/crates/tinyflows-drivers/src/catalog/drafts.rs b/crates/tinyflows-drivers/src/catalog/drafts.rs index bc653a73..d738c77e 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts.rs @@ -9,7 +9,7 @@ use anyhow::{Context, Result, bail}; use serde_json::{Value, json}; use tinyflows_catalog::{DraftOrigin, FlowDraft}; -use tinystoragedrivers_core::{DocumentStoreExt, Precondition, Query, Sort, Versioned}; +use tinyflows_storage::{DocumentStoreExt, Precondition, Query, Sort, Versioned}; use super::{ DRAFTS, FlowCatalogDocuments, compare_and_swap, instant_ms, monotonic_ms, new_id, next_stamp, diff --git a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs index b96e9539..14b54bf7 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::catalog::test_support::{catalog, catalog_in}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; fn graph() -> Value { json!({ "nodes": [{ "id": "t", "kind": "trigger" }], "edges": [] }) diff --git a/crates/tinyflows-drivers/src/catalog/import.rs b/crates/tinyflows-drivers/src/catalog/import.rs index e3ad57ab..82bdfdf4 100644 --- a/crates/tinyflows-drivers/src/catalog/import.rs +++ b/crates/tinyflows-drivers/src/catalog/import.rs @@ -17,7 +17,7 @@ use anyhow::{Context, Result, bail}; use serde_json::{Value, json}; use std::sync::Arc; use tinyflows_catalog::{Flow, FlowDraft, FlowRevision, FlowRun, FlowSuggestion}; -use tinystoragedrivers_core::{DocumentStore, ErrorKind, Precondition}; +use tinyflows_storage::{DocumentStore, ErrorKind, Precondition}; use super::definitions::to_doc as flow_doc; use super::drafts::{check_id, to_doc as draft_doc}; diff --git a/crates/tinyflows-drivers/src/catalog/kv.rs b/crates/tinyflows-drivers/src/catalog/kv.rs index 816faeda..34f10efa 100644 --- a/crates/tinyflows-drivers/src/catalog/kv.rs +++ b/crates/tinyflows-drivers/src/catalog/kv.rs @@ -9,7 +9,7 @@ use anyhow::{Context, Result}; use serde_json::Value; -use tinystoragedrivers_core::Kv; +use tinyflows_storage::Kv; use super::FlowCatalogDocuments; use crate::storage::{KV_OWNER, catalog_namespace}; diff --git a/crates/tinyflows-drivers/src/catalog/kv_tests.rs b/crates/tinyflows-drivers/src/catalog/kv_tests.rs index e96e2b96..112d4510 100644 --- a/crates/tinyflows-drivers/src/catalog/kv_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/kv_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::catalog::test_support::{catalog, catalog_in}; use serde_json::json; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn values_round_trip_per_namespace() { diff --git a/crates/tinyflows-drivers/src/catalog/lineage.rs b/crates/tinyflows-drivers/src/catalog/lineage.rs index 946bd9f1..3e7ec710 100644 --- a/crates/tinyflows-drivers/src/catalog/lineage.rs +++ b/crates/tinyflows-drivers/src/catalog/lineage.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use anyhow::Result; use serde_json::Value; -use tinystoragedrivers_core::{DocumentStore, DocumentStoreExt, ErrorKind, Query, Versioned}; +use tinyflows_storage::{DocumentStore, DocumentStoreExt, ErrorKind, Query, Versioned}; use super::{DEFINITIONS, text}; diff --git a/crates/tinyflows-drivers/src/catalog/lineage_tests.rs b/crates/tinyflows-drivers/src/catalog/lineage_tests.rs index 3233acd2..1fb8f8c8 100644 --- a/crates/tinyflows-drivers/src/catalog/lineage_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/lineage_tests.rs @@ -1,7 +1,7 @@ use super::*; use serde_json::json; use std::collections::HashMap; -use tinystoragedrivers_core::Precondition; +use tinyflows_storage::Precondition; use crate::catalog::test_support::{catalog, trigger_graph}; use crate::catalog::{FlowCatalogDocuments, RUNS}; diff --git a/crates/tinyflows-drivers/src/catalog/migrate.rs b/crates/tinyflows-drivers/src/catalog/migrate.rs index 5f12c022..5554f8d4 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate.rs @@ -8,7 +8,7 @@ //! corrupt record stays corrupt (and visible) instead of being lost. use serde_json::{Map, Value}; -use tinystoragedrivers_core::{Schema, ScopeClass, SourceRecord, Step, TargetRecord}; +use tinyflows_storage::{Schema, ScopeClass, SourceRecord, Step, TargetRecord}; use super::{ DEFINITIONS, DRAFTS, FlowCatalogDocuments, REVISIONS, RUNS, SCHEMA_VERSION, SUGGESTIONS, diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index c18399ce..24bf5b6f 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::catalog::FlowCatalogDocuments; use serde_json::json; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStoreExt, MemoryStorage, MigrateOptions, Precondition, Query, Registry, Scope, ScopedStorage, StorageBackend, WriteOptions, }; diff --git a/crates/tinyflows-drivers/src/catalog/mod.rs b/crates/tinyflows-drivers/src/catalog/mod.rs index 70e5c927..4299f9f4 100644 --- a/crates/tinyflows-drivers/src/catalog/mod.rs +++ b/crates/tinyflows-drivers/src/catalog/mod.rs @@ -65,7 +65,7 @@ use std::sync::Arc; use anyhow::{Context, Result}; use chrono::{DateTime, SecondsFormat, Utc}; use serde_json::Value; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ CollectionSpec, DocumentStore, ErrorKind, IndexSpec, Meta, Precondition, Repo, StorageError, Versioned, WriteOptions, }; @@ -150,7 +150,7 @@ impl FlowCatalogDocuments { .get_or_try_init(|| async { for spec in Self::collections() .into_iter() - .chain(tinystoragedrivers_core::primitives::collection_specs()) + .chain(tinyflows_storage::primitives::collection_specs()) { self.docs .ensure_collection(&spec) @@ -204,7 +204,7 @@ pub(crate) async fn put_doc( id: &str, doc: Value, precondition: Precondition, -) -> std::result::Result { +) -> std::result::Result { docs.put_with( collection, id, diff --git a/crates/tinyflows-drivers/src/catalog/revisions.rs b/crates/tinyflows-drivers/src/catalog/revisions.rs index ac78d659..e98abda4 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use tinyflows::model::WorkflowGraph; use tinyflows_catalog::{Flow, FlowRevision}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStore, DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Sort, Versioned, }; diff --git a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs index 53429cdf..9583143c 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs @@ -1,6 +1,6 @@ use super::*; use std::sync::Arc; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition}; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; diff --git a/crates/tinyflows-drivers/src/catalog/runs.rs b/crates/tinyflows-drivers/src/catalog/runs.rs index cc1f4e7b..459cb5b8 100644 --- a/crates/tinyflows-drivers/src/catalog/runs.rs +++ b/crates/tinyflows-drivers/src/catalog/runs.rs @@ -6,7 +6,7 @@ use anyhow::{Context, Result, bail}; use serde_json::{Value, json}; use tinyflows_catalog::{FlowRun, FlowRunStep}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Sort, Versioned, }; diff --git a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs index c8dc0428..bc1b5765 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs @@ -19,7 +19,7 @@ fn step(node: &str) -> FlowRunStep { async fn two_handles_on_one_backend_agree_on_a_run_lifecycle() { use crate::catalog::FlowStateDocuments; use tinyflows::caps::StateStore; - use tinystoragedrivers_core::MemoryStorage; + use tinyflows_storage::MemoryStorage; let storage = MemoryStorage::new(); let a = crate::catalog::test_support::catalog_in(&storage, "tenant"); diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery.rs index 30f98a45..4ffc782a 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use serde_json::json; -use tinystoragedrivers_core::{DocumentStoreExt, ErrorKind, Filter, Query, Sort, Versioned}; +use tinyflows_storage::{DocumentStoreExt, ErrorKind, Filter, Query, Sort, Versioned}; use super::lineage::{belongs_in, live_flows, reclaim}; use super::runs::{parked_before, set_finished, status}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs index b6a23d68..0a552306 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs @@ -1,11 +1,11 @@ use super::*; use tinyflows_catalog::FlowRunStep; -use tinystoragedrivers_core::{MemoryStorage, StorageBackend}; +use tinyflows_storage::{MemoryStorage, StorageBackend}; async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); let scoped = storage - .for_scope(&tinystoragedrivers_core::Scope::local()) + .for_scope(&tinyflows_storage::Scope::local()) .unwrap(); let catalog = FlowCatalogDocuments::new(scoped.documents().clone()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ @@ -32,7 +32,7 @@ async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { async fn legacy_parked() -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); let scoped = storage - .for_scope(&tinystoragedrivers_core::Scope::local()) + .for_scope(&tinyflows_storage::Scope::local()) .unwrap(); let catalog = FlowCatalogDocuments::new(scoped.documents().clone()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ diff --git a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs index df61c67c..2faad879 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs @@ -1,7 +1,7 @@ use super::*; use serde_json::json; use std::sync::Arc; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition}; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; @@ -98,12 +98,12 @@ async fn an_unchanged_flow_lists_its_runs() { /// The continuity rule `list_flow_runs` re-checks after its query. #[test] fn continuity_follows_the_incarnation_or_the_pre_fence_version() { - use tinystoragedrivers_core::Version; + use tinyflows_storage::Version; let def = |doc: serde_json::Value, version: u64| Versioned { id: "f".to_string(), version: Version(version), doc, - meta: tinystoragedrivers_core::Meta::unknown(), + meta: tinyflows_storage::Meta::unknown(), }; let fenced = def(json!({ "incarnation": "a" }), 1); assert!(still_the_flow( diff --git a/crates/tinyflows-drivers/src/catalog/state.rs b/crates/tinyflows-drivers/src/catalog/state.rs index fd36cdbd..23804d36 100644 --- a/crates/tinyflows-drivers/src/catalog/state.rs +++ b/crates/tinyflows-drivers/src/catalog/state.rs @@ -8,7 +8,7 @@ use serde_json::Value; use tinyflows::caps::StateStore; use tinyflows::error::{EngineError, Result}; use tinyflows::nodes::control_flow::dedup_settle::DedupKv; -use tinystoragedrivers_core::Blocking; +use tinyflows_storage::Blocking; use super::FlowCatalogDocuments; diff --git a/crates/tinyflows-drivers/src/catalog/state_tests.rs b/crates/tinyflows-drivers/src/catalog/state_tests.rs index 9d1ace69..1521daef 100644 --- a/crates/tinyflows-drivers/src/catalog/state_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/state_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::catalog::test_support::catalog; use serde_json::json; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capabilities, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Result as StorageResult, Sort, StorageError, Version, Versioned, }; @@ -15,7 +15,7 @@ impl DeniedStore { } } -#[tinystoragedrivers_core::async_trait] +#[tinyflows_storage::async_trait] impl DocumentStore for DeniedStore { fn capabilities(&self) -> Capabilities { Capabilities::none() diff --git a/crates/tinyflows-drivers/src/catalog/suggestions.rs b/crates/tinyflows-drivers/src/catalog/suggestions.rs index b7de8eaa..5c7b9dd9 100644 --- a/crates/tinyflows-drivers/src/catalog/suggestions.rs +++ b/crates/tinyflows-drivers/src/catalog/suggestions.rs @@ -3,7 +3,7 @@ use anyhow::{Context, Result}; use serde_json::{Value, json}; use tinyflows_catalog::{FlowSuggestion, SuggestionStatus}; -use tinystoragedrivers_core::{Filter, Query, Sort, Versioned}; +use tinyflows_storage::{Filter, Query, Sort, Versioned}; use super::{ FlowCatalogDocuments, SUGGESTIONS, compare_and_swap, instant_ms, int, required, rfc3339, diff --git a/crates/tinyflows-drivers/src/catalog/test_support.rs b/crates/tinyflows-drivers/src/catalog/test_support.rs index f300e05b..31494908 100644 --- a/crates/tinyflows-drivers/src/catalog/test_support.rs +++ b/crates/tinyflows-drivers/src/catalog/test_support.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use tinyflows::model::{Node, NodeKind, WorkflowGraph}; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; use super::FlowCatalogDocuments; @@ -105,16 +105,16 @@ impl Interposed { } } -#[tinystoragedrivers_core::async_trait] +#[tinyflows_storage::async_trait] impl DocumentStore for Interposed { - fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + fn capabilities(&self) -> tinyflows_storage::Capabilities { self.inner.capabilities() } async fn ensure_collection( &self, - spec: &tinystoragedrivers_core::CollectionSpec, - ) -> tinystoragedrivers_core::Result<()> { + spec: &tinyflows_storage::CollectionSpec, + ) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -122,8 +122,8 @@ impl DocumentStore for Interposed { &self, collection: &str, id: &str, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { self.maybe_interject("get", collection).await; self.inner.get(collection, id).await @@ -134,8 +134,8 @@ impl DocumentStore for Interposed { collection: &str, id: &str, doc: serde_json::Value, - precondition: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { self.maybe_interject("put", collection).await; self.inner.put(collection, id, doc, precondition).await } @@ -145,9 +145,9 @@ impl DocumentStore for Interposed { collection: &str, id: &str, doc: serde_json::Value, - precondition: tinystoragedrivers_core::Precondition, - options: tinystoragedrivers_core::WriteOptions, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + options: tinyflows_storage::WriteOptions, + ) -> tinyflows_storage::Result { self.maybe_interject("put_with", collection).await; self.inner .put_with(collection, id, doc, precondition, options) @@ -158,8 +158,8 @@ impl DocumentStore for Interposed { &self, collection: &str, id: &str, - precondition: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { self.maybe_interject("delete", collection).await; self.inner.delete(collection, id, precondition).await } @@ -167,9 +167,9 @@ impl DocumentStore for Interposed { async fn query( &self, collection: &str, - query: &tinystoragedrivers_core::Query, - ) -> tinystoragedrivers_core::Result< - tinystoragedrivers_core::Page>, + query: &tinyflows_storage::Query, + ) -> tinyflows_storage::Result< + tinyflows_storage::Page>, > { self.maybe_interject("query", collection).await; self.inner.query(collection, query).await @@ -178,32 +178,32 @@ impl DocumentStore for Interposed { async fn count( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + filter: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } async fn delete_where( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + filter: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.delete_where(collection, filter).await } async fn claim( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - sort: &[tinystoragedrivers_core::Sort], + filter: &tinyflows_storage::Filter, + sort: &[tinyflows_storage::Sort], patch: &serde_json::Value, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { self.inner.claim(collection, filter, sort, patch).await } - async fn drop_collection(&self, collection: &str) -> tinystoragedrivers_core::Result<()> { + async fn drop_collection(&self, collection: &str) -> tinyflows_storage::Result<()> { self.inner.drop_collection(collection).await } } diff --git a/crates/tinyflows-drivers/src/checkpoint.rs b/crates/tinyflows-drivers/src/checkpoint.rs index fccb361b..5705b56f 100644 --- a/crates/tinyflows-drivers/src/checkpoint.rs +++ b/crates/tinyflows-drivers/src/checkpoint.rs @@ -44,7 +44,7 @@ use tinyflows::graph::ids::CheckpointId; use tinyflows::graph::{ Checkpoint, CheckpointConfig, CheckpointMetadata, CheckpointTuple, Checkpointer, PendingWrite, }; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStore, DocumentStoreExt, ErrorKind, Filter, Query, Repo, Sort, Versioned, WriteOptions, }; use tokio::sync::OnceCell; diff --git a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs index bfd949ba..84576d40 100644 --- a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs @@ -2,9 +2,9 @@ use super::*; use serde_json::json; use std::sync::Arc; use tinyflows::graph::ids::NodeId; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; #[cfg(feature = "catalog")] -use tinystoragedrivers_core::{Precondition, WriteOptions}; +use tinyflows_storage::{Precondition, WriteOptions}; #[tokio::test] async fn legacy_write_import_cas_preserves_existing_task_values() { diff --git a/crates/tinyflows-drivers/src/checkpoint_layout.rs b/crates/tinyflows-drivers/src/checkpoint_layout.rs index 54f6eef0..29d8802f 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout.rs @@ -14,7 +14,7 @@ use serde_json::Value; use tinyflows::graph::CheckpointConfig; use tinyflows::graph::PendingWrite; use tinyflows::graph::error::{GraphError, Result}; -use tinystoragedrivers_core::{CollectionSpec, IndexSpec, Key, StorageError}; +use tinyflows_storage::{CollectionSpec, IndexSpec, Key, StorageError}; /// The body version of every `graph.*` document (tinyagents' `SCHEMA_VERSION`). pub(crate) const SCHEMA_VERSION: u32 = 4; diff --git a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs index f87d267a..d1cbf792 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs @@ -8,7 +8,7 @@ use crate::checkpoint::DriverCheckpointer; use serde_json::json; use std::sync::Arc; use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Precondition, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition, Scope, StorageBackend}; fn docs() -> Arc { Arc::clone( @@ -50,7 +50,7 @@ async fn documents_migrated_by_tinyagents_step_are_read_back() { json!({"ns": "flows", "thread": "t", "subgraph": "", "seq": 0, "checkpoint_id": "c1", "checkpoint": record("t", "c1", &[])}), Precondition::Absent, - tinystoragedrivers_core::WriteOptions::new().schema(4), + tinyflows_storage::WriteOptions::new().schema(4), ) .await .unwrap(); @@ -60,7 +60,7 @@ async fn documents_migrated_by_tinyagents_step_are_read_back() { json!({"ns": "flows", "thread": "t", "subgraph": "n/sub", "seq": 1, "checkpoint_id": "c1", "checkpoint": record("t", "c1", &["sub"])}), Precondition::Absent, - tinystoragedrivers_core::WriteOptions::new().schema(4), + tinyflows_storage::WriteOptions::new().schema(4), ) .await .unwrap(); diff --git a/crates/tinyflows-drivers/src/checkpoint_tests.rs b/crates/tinyflows-drivers/src/checkpoint_tests.rs index 616d3ec7..f00a0b3c 100644 --- a/crates/tinyflows-drivers/src/checkpoint_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_tests.rs @@ -6,7 +6,7 @@ use super::*; use crate::checkpoint_layout::{checkpoint_id_key, subgraph_key, thread_id_key}; use serde_json::json; use tinyflows::graph::ids::NodeId; -use tinystoragedrivers_core::{MemoryStorage, Precondition, Scope, StorageBackend, StorageError}; +use tinyflows_storage::{MemoryStorage, Precondition, Scope, StorageBackend, StorageError}; fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( diff --git a/crates/tinyflows-drivers/src/ids.rs b/crates/tinyflows-drivers/src/ids.rs index d4bf7b62..2264f71a 100644 --- a/crates/tinyflows-drivers/src/ids.rs +++ b/crates/tinyflows-drivers/src/ids.rs @@ -3,7 +3,7 @@ use std::sync::atomic::{AtomicI64, Ordering}; use chrono::Utc; -use tinystoragedrivers_core::Key; +use tinyflows_storage::Key; /// Now in epoch milliseconds, never repeating within the process: creations /// in quick succession still get distinct, increasing times, which is what diff --git a/crates/tinyflows-drivers/src/schedule/admission.rs b/crates/tinyflows-drivers/src/schedule/admission.rs index 0162de9c..61cbbe5c 100644 --- a/crates/tinyflows-drivers/src/schedule/admission.rs +++ b/crates/tinyflows-drivers/src/schedule/admission.rs @@ -4,7 +4,7 @@ use super::{ADMISSION, CAS_ATTEMPTS, CronDocuments, JOBS, storage_error}; use anyhow::{Result, anyhow, bail}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capability, DocumentStoreExt, ErrorKind, Precondition, Query, Versioned, WriteOp, WriteResult, }; @@ -193,7 +193,7 @@ impl CronDocuments { created_ms: i64, ) -> Result<()> { let mut options = - tinystoragedrivers_core::WriteOptions::new().schema(super::SCHEMA_VERSION); + tinyflows_storage::WriteOptions::new().schema(super::SCHEMA_VERSION); if created_ms > 0 { options = options.created_ms(created_ms); } @@ -306,7 +306,7 @@ impl CronDocuments { }; if !owns { return Err(storage_error( - tinystoragedrivers_core::StorageError::conflict("cron job incarnation changed"), + tinyflows_storage::StorageError::conflict("cron job incarnation changed"), )); } let current = self.docs.get(JOBS, id).await.map_err(storage_error)?; @@ -315,7 +315,7 @@ impl CronDocuments { (Some(old), Some(current)) if old.doc == current.doc => current.unchanged(), _ => { return Err(storage_error( - tinystoragedrivers_core::StorageError::conflict("cron job changed"), + tinyflows_storage::StorageError::conflict("cron job changed"), )); } }; @@ -355,6 +355,6 @@ impl CronDocuments { pub(super) fn conflict(error: &anyhow::Error) -> bool { // `storage_error` retains the storage error as an anyhow source below. error - .downcast_ref::() + .downcast_ref::() .is_some_and(|error| error.kind() == ErrorKind::Conflict) } diff --git a/crates/tinyflows-drivers/src/schedule/admission_tests.rs b/crates/tinyflows-drivers/src/schedule/admission_tests.rs index 004be5d6..dbab8fb1 100644 --- a/crates/tinyflows-drivers/src/schedule/admission_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/admission_tests.rs @@ -4,7 +4,7 @@ use super::{ test_support::{daily, docs_in}, }; use tinyflows_schedule::CronJobPatch; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn an_existing_unlimited_handle_cannot_bypass_installed_policy() { @@ -168,7 +168,7 @@ async fn transaction_refusal_and_completion_failure_leave_jobs_unchanged() { async fn malformed_admission_and_capability_downgrades_fail_closed() { use super::test_support::FaultDocuments; use std::sync::{Arc, atomic::AtomicBool}; - use tinystoragedrivers_core::Precondition; + use tinyflows_storage::Precondition; let backend = MemoryStorage::new(); let a = CronDocuments::new(docs_in(&backend, "downgrade")); a.set_max_active_jobs(Some(1)).await.unwrap(); @@ -302,7 +302,7 @@ async fn admission_wire_form_lists_members_with_their_job_ids() { #[tokio::test] async fn policy_installation_reconciles_legacy_rows_written_before_adoption() { - use tinystoragedrivers_core::Precondition; + use tinyflows_storage::Precondition; let a = super::test_support::store(); let first = a.add_shell_job(None, daily(), "upgraded").await.unwrap(); // An older writer during a rolling upgrade did not know the ledger. diff --git a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs index 6a974b81..56eaae2a 100644 --- a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs @@ -4,7 +4,7 @@ use super::{ test_support::{daily, docs_in}, }; use chrono::Duration; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn completed_receipt_cannot_remove_a_recreated_job() { diff --git a/crates/tinyflows-drivers/src/schedule/claims.rs b/crates/tinyflows-drivers/src/schedule/claims.rs index 820a6578..614133f4 100644 --- a/crates/tinyflows-drivers/src/schedule/claims.rs +++ b/crates/tinyflows-drivers/src/schedule/claims.rs @@ -31,7 +31,7 @@ use chrono::{DateTime, Duration, Utc}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tinyflows_schedule::CronJob; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capability, ErrorKind, Key, Leases, Precondition, Versioned, WriteOp, primitives, }; diff --git a/crates/tinyflows-drivers/src/schedule/claims/acquisition.rs b/crates/tinyflows-drivers/src/schedule/claims/acquisition.rs index 12453e42..54dca263 100644 --- a/crates/tinyflows-drivers/src/schedule/claims/acquisition.rs +++ b/crates/tinyflows-drivers/src/schedule/claims/acquisition.rs @@ -1,5 +1,5 @@ use super::*; -use tinystoragedrivers_core::Lease; +use tinyflows_storage::Lease; use uuid::Uuid; impl CronDocuments { diff --git a/crates/tinyflows-drivers/src/schedule/claims/completion.rs b/crates/tinyflows-drivers/src/schedule/claims/completion.rs index 872939d9..e08452b4 100644 --- a/crates/tinyflows-drivers/src/schedule/claims/completion.rs +++ b/crates/tinyflows-drivers/src/schedule/claims/completion.rs @@ -9,7 +9,7 @@ use super::{ use anyhow::{Result, bail}; use chrono::{DateTime, Utc}; use tinyflows_schedule::{DeliveryStatus, Schedule, next_run_for_schedule, truncate_cron_output}; -use tinystoragedrivers_core::{Capability, Precondition, WriteOp}; +use tinyflows_storage::{Capability, Precondition, WriteOp}; /// The outcome and invocation identity to persist for a claimed run. #[derive(Debug, Clone)] diff --git a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs index b12ed0a9..f0e483a6 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs @@ -4,7 +4,7 @@ use super::test_support::{daily, docs_in, store}; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::Schedule; -use tinystoragedrivers_core::{DocumentStoreExt, MemoryStorage, Query, primitives}; +use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Query, primitives}; async fn lease_count(store: &CronDocuments) -> usize { store diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index 71ac371b..5dac3a76 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -6,7 +6,7 @@ use super::test_support::{FaultDocuments, daily, docs_in}; use chrono::Duration; use std::sync::{Arc, atomic::AtomicBool}; use tinyflows_schedule::CronJobPatch; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; fn fault_store( backend: &MemoryStorage, diff --git a/crates/tinyflows-drivers/src/schedule/claims_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_tests.rs index c9aebab9..4171e04e 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_tests.rs @@ -4,7 +4,7 @@ use super::{ test_support::{daily, docs_in}, }; use chrono::Duration; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn two_handles_cannot_claim_the_same_occurrence() { @@ -282,7 +282,7 @@ async fn release_and_restart_recovery_issue_fresh_tokens() { #[tokio::test] async fn invalid_inputs_and_malformed_stored_leases_fail_closed() { use serde_json::json; - use tinystoragedrivers_core::Precondition; + use tinyflows_storage::Precondition; let a = super::test_support::store(); let job = a.add_shell_job(None, daily(), "echo").await.unwrap(); let now = job.next_run; @@ -323,12 +323,12 @@ async fn invalid_inputs_and_malformed_stored_leases_fail_closed() { .is_err() ); // The claim's lease record, tampered with. - let key = tinystoragedrivers_core::Key::of([ + let key = tinyflows_storage::Key::of([ job.id.as_str(), job.next_run.timestamp_millis().to_string().as_str(), ]) .unwrap(); - let id = tinystoragedrivers_core::Key::of(["cron", key.as_str()]).unwrap(); + let id = tinyflows_storage::Key::of(["cron", key.as_str()]).unwrap(); a.docs .put( "tsd.leases", @@ -437,7 +437,7 @@ async fn completion_already_in_flight_cannot_write_into_a_recreated_job() { #[tokio::test] async fn a_claim_is_a_tsd_lease_and_run_numbers_a_tsd_counter() { use serde_json::Value; - use tinystoragedrivers_core::Key; + use tinyflows_storage::Key; let a = super::test_support::store(); let job = a.add_shell_job(None, daily(), "echo").await.unwrap(); let claim = a diff --git a/crates/tinyflows-drivers/src/schedule/codec.rs b/crates/tinyflows-drivers/src/schedule/codec.rs index 904dffa7..1a576b1a 100644 --- a/crates/tinyflows-drivers/src/schedule/codec.rs +++ b/crates/tinyflows-drivers/src/schedule/codec.rs @@ -14,7 +14,7 @@ use serde_json::{Map, Value, json}; use tinyflows_schedule::{ CronJob, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobType, Schedule, SessionTarget, }; -use tinystoragedrivers_core::Versioned; +use tinyflows_storage::Versioned; /// The string field `field` of `doc`. pub(super) fn text<'a>(doc: &'a Value, field: &str) -> Option<&'a str> { diff --git a/crates/tinyflows-drivers/src/schedule/codec_tests.rs b/crates/tinyflows-drivers/src/schedule/codec_tests.rs index 923d04bd..89dcd05a 100644 --- a/crates/tinyflows-drivers/src/schedule/codec_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/codec_tests.rs @@ -1,7 +1,7 @@ use super::*; use serde_json::json; -use tinystoragedrivers_core::{Meta, Version}; +use tinyflows_storage::{Meta, Version}; fn stored(doc: Value) -> Versioned { Versioned { diff --git a/crates/tinyflows-drivers/src/schedule/import.rs b/crates/tinyflows-drivers/src/schedule/import.rs index ab8e933d..d4e6fdfb 100644 --- a/crates/tinyflows-drivers/src/schedule/import.rs +++ b/crates/tinyflows-drivers/src/schedule/import.rs @@ -24,7 +24,7 @@ use anyhow::{Context, Result}; use tinyflows_schedule::{CronJob, CronRun, JobType}; -use tinystoragedrivers_core::{Counters, ErrorKind, Filter, Precondition, Query, WriteOptions}; +use tinyflows_storage::{Counters, ErrorKind, Filter, Precondition, Query, WriteOptions}; use super::codec::{INCARNATION, run_id, run_to_doc}; use super::jobs::{flow_job_id, new_job_doc, runs_of}; @@ -165,7 +165,7 @@ impl CronDocuments { /// Whether `error` is the storage refusing to write an id that is taken. fn is_taken(error: &anyhow::Error) -> bool { error - .downcast_ref::() + .downcast_ref::() .is_some_and(|error| matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists)) } diff --git a/crates/tinyflows-drivers/src/schedule/import_tests.rs b/crates/tinyflows-drivers/src/schedule/import_tests.rs index 0ca9488a..edf205b5 100644 --- a/crates/tinyflows-drivers/src/schedule/import_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/import_tests.rs @@ -4,7 +4,7 @@ use crate::schedule::job_not_found; use crate::schedule::test_support::{channel_origin, daily, docs_in, store}; use chrono::{TimeZone, Utc}; use tinyflows_schedule::{DeliveryStatus, Schedule}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; /// The JSON form, for comparing jobs (`CronJob` is not `PartialEq`). fn j(value: &T) -> serde_json::Value { diff --git a/crates/tinyflows-drivers/src/schedule/jobs.rs b/crates/tinyflows-drivers/src/schedule/jobs.rs index 59818ab5..8e74e1d5 100644 --- a/crates/tinyflows-drivers/src/schedule/jobs.rs +++ b/crates/tinyflows-drivers/src/schedule/jobs.rs @@ -11,7 +11,7 @@ use tinyflows_schedule::{ SessionTarget, next_run_for_schedule, schedule_cron_expression, validate_agent_schedule, validate_schedule, }; -use tinystoragedrivers_core::{DocumentStoreExt, Filter, Query, Sort, Versioned, WriteResult}; +use tinyflows_storage::{DocumentStoreExt, Filter, Query, Sort, Versioned, WriteResult}; use uuid::Uuid; use super::codec::{INCARNATION, doc_to_job, incarnation, job_to_doc, set_next_run, text}; diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs index 25231d73..45e0a27c 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs @@ -4,7 +4,7 @@ use chrono::{Duration, Utc}; use serde_json::Value; use std::sync::Arc; use tinyflows_schedule::Schedule; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capabilities, CollectionSpec, DocumentStore, Filter, MemoryStorage, Page, Precondition, Query, Scope, Sort, StorageBackend, Version, Versioned, WriteOp, WriteResult, }; @@ -27,7 +27,7 @@ impl DocumentStore for HeldClaimWrite { async fn ensure_collection( &self, spec: &CollectionSpec, - ) -> tinystoragedrivers_core::Result<()> { + ) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -35,7 +35,7 @@ impl DocumentStore for HeldClaimWrite { &self, collection: &str, id: &str, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.get(collection, id).await } @@ -45,7 +45,7 @@ impl DocumentStore for HeldClaimWrite { id: &str, doc: Value, precondition: Precondition, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.put(collection, id, doc, precondition).await } @@ -54,7 +54,7 @@ impl DocumentStore for HeldClaimWrite { collection: &str, id: &str, precondition: Precondition, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.delete(collection, id, precondition).await } @@ -62,7 +62,7 @@ impl DocumentStore for HeldClaimWrite { &self, collection: &str, query: &Query, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.query(collection, query).await } @@ -70,7 +70,7 @@ impl DocumentStore for HeldClaimWrite { &self, collection: &str, filter: &Filter, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } @@ -78,7 +78,7 @@ impl DocumentStore for HeldClaimWrite { &self, collection: &str, filter: &Filter, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.delete_where(collection, filter).await } @@ -88,14 +88,14 @@ impl DocumentStore for HeldClaimWrite { filter: &Filter, sort: &[Sort], patch: &Value, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.claim(collection, filter, sort, patch).await } async fn atomic_batch( &self, ops: Vec, - ) -> tinystoragedrivers_core::Result> { + ) -> tinyflows_storage::Result> { let late_claim = ops.iter().any(|op| match op { WriteOp::Put { collection, doc, .. @@ -124,14 +124,14 @@ impl DocumentStore for HeldClaimWrite { .lose_fence_reply .swap(false, std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::backend( + return Err(tinyflows_storage::StorageError::backend( "injected lost admission fence response", )); } Ok(result) } - async fn drop_collection(&self, collection: &str) -> tinystoragedrivers_core::Result<()> { + async fn drop_collection(&self, collection: &str) -> tinyflows_storage::Result<()> { self.inner.drop_collection(collection).await } } diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs index 6ae101c3..6ae138ee 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs @@ -2,7 +2,7 @@ use super::test_support::docs_in; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::{CronJobPatch, DeliveryStatus, Schedule}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn reconciles_manual_claim_receipt_after_an_ambiguous_claim_reply() { diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index 44cea5f2..8434c030 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -21,7 +21,7 @@ //! moves it into `tsd.counters` once per scope after `Registry::migrate`. use serde_json::{Map, Value, json}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Counters, Migrations, Owner, Precondition, Result, Schema, ScopeClass, ScopedStorage, SourceRecord, Step, TargetRecord, }; @@ -253,7 +253,7 @@ pub async fn import_legacy_cron_counters(storage: &ScopedStorage) -> Result { - let Some(tinystoragedrivers_core::WriteResult::Put { version }) = results.first() + let Some(tinyflows_storage::WriteResult::Put { version }) = results.first() else { anyhow::bail!("cron store: invalid atomic write result"); }; diff --git a/crates/tinyflows-drivers/src/schedule/mod_tests.rs b/crates/tinyflows-drivers/src/schedule/mod_tests.rs index 2c6aabce..2fbc33f1 100644 --- a/crates/tinyflows-drivers/src/schedule/mod_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/mod_tests.rs @@ -3,7 +3,7 @@ use super::*; use super::test_support::{daily, docs_in, store}; use chrono::Utc; use serde_json::json; -use tinystoragedrivers_core::{MemoryStorage, Precondition}; +use tinyflows_storage::{MemoryStorage, Precondition}; #[test] fn limits_default_like_the_sqlite_store_and_clamp_to_one() { @@ -91,7 +91,7 @@ async fn compare_and_swap_skips_missing_documents_and_declined_changes() { #[tokio::test] async fn legacy_policy_documents_move_into_jobs_without_losing_seeded_fields() { - use tinystoragedrivers_core::CollectionSpec; + use tinyflows_storage::CollectionSpec; let store = store(); let job = store .add_shell_job(Some("system".into()), daily(), "system:seeded") diff --git a/crates/tinyflows-drivers/src/schedule/policy.rs b/crates/tinyflows-drivers/src/schedule/policy.rs index 9767769b..97aeb132 100644 --- a/crates/tinyflows-drivers/src/schedule/policy.rs +++ b/crates/tinyflows-drivers/src/schedule/policy.rs @@ -1,6 +1,6 @@ //! Upgrade legacy host policies through CAS; never lose fields or delete first. use super::*; -use tinystoragedrivers_core::{DocumentStoreExt, ErrorKind, Query}; +use tinyflows_storage::{DocumentStoreExt, ErrorKind, Query}; impl CronDocuments { pub(super) async fn migrate_policies(&self) -> Result<()> { diff --git a/crates/tinyflows-drivers/src/schedule/removal.rs b/crates/tinyflows-drivers/src/schedule/removal.rs index 8c5a49f8..1f6309ed 100644 --- a/crates/tinyflows-drivers/src/schedule/removal.rs +++ b/crates/tinyflows-drivers/src/schedule/removal.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use serde_json::Value; -use tinystoragedrivers_core::{Capability, DocumentStoreExt, Filter, Query, Versioned}; +use tinyflows_storage::{Capability, DocumentStoreExt, Filter, Query, Versioned}; use super::codec::incarnation; use super::jobs::runs_of; @@ -59,7 +59,7 @@ impl CronDocuments { { Ok(results) => matches!( results.first(), - Some(tinystoragedrivers_core::WriteResult::Delete { removed: true }) + Some(tinyflows_storage::WriteResult::Delete { removed: true }) ), Err(error) if super::admission::conflict(&error) => false, Err(error) => return Err(error).context("Failed to delete cron job"), @@ -105,7 +105,7 @@ impl CronDocuments { Ok(results) if matches!( results.first(), - Some(tinystoragedrivers_core::WriteResult::Delete { removed: true }) + Some(tinyflows_storage::WriteResult::Delete { removed: true }) ) => { self.delete_runs_of(stored).await?; diff --git a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs index eb542720..b5e86c9d 100644 --- a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs @@ -4,7 +4,7 @@ use super::super::{ test_support::{FaultDocuments, daily, docs_in}, }; use std::sync::{Arc, atomic::AtomicBool}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[tokio::test] async fn nontransactional_deduplication_and_clear_keep_legacy_behavior() { @@ -69,7 +69,7 @@ async fn deduplication_frees_an_enabled_slot_and_leaves_disabled_jobs_uncounted( #[tokio::test] async fn legacy_rows_are_imported_before_admission_policy_is_installed() { - use tinystoragedrivers_core::Precondition; + use tinyflows_storage::Precondition; let backend = MemoryStorage::new(); let a = CronDocuments::new(Arc::new(FaultDocuments { inner: docs_in(&backend, "legacy-admission"), @@ -113,7 +113,7 @@ async fn legacy_rows_are_imported_before_admission_policy_is_installed() { #[tokio::test] async fn a_legacy_row_without_enabled_is_imported_as_disabled() { - use tinystoragedrivers_core::Precondition; + use tinyflows_storage::Precondition; let backend = MemoryStorage::new(); let a = CronDocuments::new(Arc::new(FaultDocuments { inner: docs_in(&backend, "legacy-disabled"), diff --git a/crates/tinyflows-drivers/src/schedule/runs.rs b/crates/tinyflows-drivers/src/schedule/runs.rs index 0df13306..7d2106e6 100644 --- a/crates/tinyflows-drivers/src/schedule/runs.rs +++ b/crates/tinyflows-drivers/src/schedule/runs.rs @@ -7,7 +7,7 @@ use serde_json::Value; use tinyflows_schedule::{ CronJob, CronRun, DeliveryStatus, Schedule, next_run_for_schedule, truncate_cron_output, }; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Counters, DocumentStoreExt, ErrorKind, Filter, Precondition, Query, Sort, Versioned, }; @@ -225,7 +225,7 @@ impl CronDocuments { &run_id(seq), doc, Precondition::Absent, - tinystoragedrivers_core::WriteOptions::new().schema(SCHEMA_VERSION), + tinyflows_storage::WriteOptions::new().schema(SCHEMA_VERSION), ) .await .map_err(storage_error) diff --git a/crates/tinyflows-drivers/src/schedule/runs_sweep_tests.rs b/crates/tinyflows-drivers/src/schedule/runs_sweep_tests.rs index b8beea8b..922e9678 100644 --- a/crates/tinyflows-drivers/src/schedule/runs_sweep_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/runs_sweep_tests.rs @@ -104,7 +104,7 @@ async fn removing_a_legacy_job_spares_a_replacements_runs() { .put(JOBS, &job.id, doc.clone(), legacy.unchanged()) .await .unwrap(); - let legacy = tinystoragedrivers_core::Versioned { + let legacy = tinyflows_storage::Versioned { id: job.id.clone(), version, doc, diff --git a/crates/tinyflows-drivers/src/schedule/runtime.rs b/crates/tinyflows-drivers/src/schedule/runtime.rs index f3ad98bb..d38990a4 100644 --- a/crates/tinyflows-drivers/src/schedule/runtime.rs +++ b/crates/tinyflows-drivers/src/schedule/runtime.rs @@ -8,7 +8,7 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use serde_json::Value; use tinyflows_schedule::{CronJob, Schedule, runtime::Completion, truncate_cron_output}; -use tinystoragedrivers_core::Versioned; +use tinyflows_storage::Versioned; fn decode(stored: &Versioned) -> Result { doc_to_job(stored) } diff --git a/crates/tinyflows-drivers/src/schedule/test_support.rs b/crates/tinyflows-drivers/src/schedule/test_support.rs index b07be5ee..32778aa1 100644 --- a/crates/tinyflows-drivers/src/schedule/test_support.rs +++ b/crates/tinyflows-drivers/src/schedule/test_support.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use tinyflows_schedule::{JobOrigin, Schedule}; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; use super::CronDocuments; @@ -63,14 +63,14 @@ pub(super) struct Faults { } impl FaultDocuments { - fn lease_write_fault(&self, collection: &str) -> tinystoragedrivers_core::Result<()> { + fn lease_write_fault(&self, collection: &str) -> tinyflows_storage::Result<()> { if collection == "tsd.leases" && self .faults .lease_writes .load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::unavailable( + return Err(tinyflows_storage::StorageError::unavailable( "fixture refuses lease writes", )); } @@ -79,27 +79,27 @@ impl FaultDocuments { } #[async_trait::async_trait] impl DocumentStore for FaultDocuments { - fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + fn capabilities(&self) -> tinyflows_storage::Capabilities { if self.transactions { self.inner.capabilities() } else { self.inner .capabilities() - .without(tinystoragedrivers_core::Capability::Transactions) + .without(tinyflows_storage::Capability::Transactions) } } async fn ensure_collection( &self, - spec: &tinystoragedrivers_core::CollectionSpec, - ) -> tinystoragedrivers_core::Result<()> { + spec: &tinyflows_storage::CollectionSpec, + ) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } async fn get( &self, c: &str, id: &str, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { if c == "cron.jobs" { let read = self @@ -113,7 +113,7 @@ impl DocumentStore for FaultDocuments { .fail_job_read .load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::unavailable( + return Err(tinyflows_storage::StorageError::unavailable( "fixture refuses this job read", )); } @@ -125,8 +125,8 @@ impl DocumentStore for FaultDocuments { c: &str, id: &str, doc: serde_json::Value, - p: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + p: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { self.lease_write_fault(c)?; self.inner.put(c, id, doc, p).await } @@ -135,9 +135,9 @@ impl DocumentStore for FaultDocuments { c: &str, id: &str, doc: serde_json::Value, - p: tinystoragedrivers_core::Precondition, - options: tinystoragedrivers_core::WriteOptions, - ) -> tinystoragedrivers_core::Result { + p: tinyflows_storage::Precondition, + options: tinyflows_storage::WriteOptions, + ) -> tinyflows_storage::Result { self.lease_write_fault(c)?; self.inner.put_with(c, id, doc, p, options).await } @@ -145,59 +145,59 @@ impl DocumentStore for FaultDocuments { &self, c: &str, id: &str, - p: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + p: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { self.lease_write_fault(c)?; self.inner.delete(c, id, p).await } async fn query( &self, c: &str, - q: &tinystoragedrivers_core::Query, - ) -> tinystoragedrivers_core::Result< - tinystoragedrivers_core::Page>, + q: &tinyflows_storage::Query, + ) -> tinyflows_storage::Result< + tinyflows_storage::Page>, > { self.inner.query(c, q).await } async fn count( &self, c: &str, - f: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + f: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.count(c, f).await } async fn delete_where( &self, c: &str, - f: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + f: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.delete_where(c, f).await } async fn claim( &self, c: &str, - f: &tinystoragedrivers_core::Filter, - s: &[tinystoragedrivers_core::Sort], + f: &tinyflows_storage::Filter, + s: &[tinyflows_storage::Sort], p: &serde_json::Value, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { self.inner.claim(c, f, s, p).await } - async fn drop_collection(&self, c: &str) -> tinystoragedrivers_core::Result<()> { + async fn drop_collection(&self, c: &str) -> tinyflows_storage::Result<()> { self.inner.drop_collection(c).await } async fn atomic_batch( &self, - ops: Vec, - ) -> tinystoragedrivers_core::Result> { + ops: Vec, + ) -> tinyflows_storage::Result> { let barrier = self.batch_barrier.lock().unwrap().take(); if let Some(barrier) = barrier { barrier.wait().await; } if self.fail_batch.load(std::sync::atomic::Ordering::SeqCst) || !self.transactions { - return Err(tinystoragedrivers_core::StorageError::unsupported( - tinystoragedrivers_core::Capability::Transactions, + return Err(tinyflows_storage::StorageError::unsupported( + tinyflows_storage::Capability::Transactions, "fixture refuses transaction", )); } diff --git a/crates/tinyflows-drivers/src/state_store.rs b/crates/tinyflows-drivers/src/state_store.rs index 3a64b0c7..91eba567 100644 --- a/crates/tinyflows-drivers/src/state_store.rs +++ b/crates/tinyflows-drivers/src/state_store.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use async_trait::async_trait; use serde_json::Value; -use tinystoragedrivers_core::{DocumentStore, Kv, StorageError, primitives}; +use tinyflows_storage::{DocumentStore, Kv, StorageError, primitives}; use tokio::sync::OnceCell; use tinyflows::caps::StateStore; diff --git a/crates/tinyflows-drivers/src/state_store_tests.rs b/crates/tinyflows-drivers/src/state_store_tests.rs index b5ecf9b7..5facfbbc 100644 --- a/crates/tinyflows-drivers/src/state_store_tests.rs +++ b/crates/tinyflows-drivers/src/state_store_tests.rs @@ -1,6 +1,6 @@ use super::*; use serde_json::json; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( diff --git a/crates/tinyflows-drivers/src/storage/kv_import.rs b/crates/tinyflows-drivers/src/storage/kv_import.rs index 85bbff25..8873aa71 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import.rs @@ -12,7 +12,7 @@ //! is never overwritten by a stale v1 row. use serde_json::Value; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ CollectionSpec, Cursor, ErrorKind, Filter, ID_FIELD, Key, Migrations, Precondition, Query, Result, ScopedStorage, Sort, primitives, }; diff --git a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs index 46aa0fb0..5907f178 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs @@ -1,6 +1,6 @@ use super::*; use serde_json::json; -use tinystoragedrivers_core::{Kv, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{Kv, MemoryStorage, Scope, StorageBackend}; fn scoped() -> ScopedStorage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() diff --git a/crates/tinyflows-drivers/src/storage/mod.rs b/crates/tinyflows-drivers/src/storage/mod.rs index b2e1f1f1..fa654802 100644 --- a/crates/tinyflows-drivers/src/storage/mod.rs +++ b/crates/tinyflows-drivers/src/storage/mod.rs @@ -24,7 +24,7 @@ mod kv_import; use std::sync::Arc; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStore, MemoryStorage, Owner, Schema, Scope, ScopedStorage, StorageBackend, }; @@ -78,7 +78,7 @@ pub fn catalog_storage_schema() -> Schema { pub fn document_only_scope( scope: Scope, documents: Arc, -) -> tinystoragedrivers_core::Result { +) -> tinyflows_storage::Result { let memory = MemoryStorage::new().for_scope(&scope)?; Ok(ScopedStorage::new( scope, diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index af0f8db1..4cd34041 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -13,7 +13,7 @@ use tinyflows::store::{ GraphHandle, WorkflowStore, apply_workflow_ops, apply_workflow_ops_observed, create_workflow, preview_workflow_ops, validate_handle, }; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; use super::test_support::store_over; diff --git a/crates/tinyflows-drivers/src/store/concurrency_tests.rs b/crates/tinyflows-drivers/src/store/concurrency_tests.rs index 5297745a..ece62d80 100644 --- a/crates/tinyflows-drivers/src/store/concurrency_tests.rs +++ b/crates/tinyflows-drivers/src/store/concurrency_tests.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Barrier}; use tinyflows::store::WorkflowStore; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; use super::test_support::{record, store, store_over}; diff --git a/crates/tinyflows-drivers/src/store/definitions.rs b/crates/tinyflows-drivers/src/store/definitions.rs index 87c60333..0f6c0135 100644 --- a/crates/tinyflows-drivers/src/store/definitions.rs +++ b/crates/tinyflows-drivers/src/store/definitions.rs @@ -3,7 +3,7 @@ use serde_json::Value; use tinyflows::store::types::{WorkflowError, WorkflowId, WorkflowRecord, WorkflowSummary}; -use tinystoragedrivers_core::{DocumentStoreExt, ErrorKind, Precondition, Query, Sort, Versioned}; +use tinyflows_storage::{DocumentStoreExt, ErrorKind, Precondition, Query, Sort, Versioned}; use super::codec::{definition_body, definition_from_body}; use super::{CAS_ATTEMPTS, DEFINITIONS, Inner, doc_id, put_doc, storage}; diff --git a/crates/tinyflows-drivers/src/store/history.rs b/crates/tinyflows-drivers/src/store/history.rs index 3d80e688..59ea7591 100644 --- a/crates/tinyflows-drivers/src/store/history.rs +++ b/crates/tinyflows-drivers/src/store/history.rs @@ -4,7 +4,7 @@ use serde_json::Value; use tinyflows::store::types::{WorkflowError, WorkflowRecord, WorkflowRevision}; use tinyflows::store::{MAX_REVISIONS, mint_revision_id}; -use tinystoragedrivers_core::{DocumentStoreExt, Filter, Precondition, Query, Sort, Versioned}; +use tinyflows_storage::{DocumentStoreExt, Filter, Precondition, Query, Sort, Versioned}; use super::codec::{revision_body, revision_record}; use super::{Inner, REVISIONS, now_ms, put_doc, stamp, storage}; diff --git a/crates/tinyflows-drivers/src/store/history_tests.rs b/crates/tinyflows-drivers/src/store/history_tests.rs index 1b042c00..f9cd83c7 100644 --- a/crates/tinyflows-drivers/src/store/history_tests.rs +++ b/crates/tinyflows-drivers/src/store/history_tests.rs @@ -6,7 +6,7 @@ use super::test_support::{record, store, store_over}; use tinyflows::store::types::WorkflowError; use tinyflows::store::{MAX_REVISIONS, WorkflowStore, require, rollback, undo_last}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; #[test] fn a_workflow_saved_for_the_first_time_has_no_history_to_go_back_to() { diff --git a/crates/tinyflows-drivers/src/store/import.rs b/crates/tinyflows-drivers/src/store/import.rs index f8a541f2..7a89e60d 100644 --- a/crates/tinyflows-drivers/src/store/import.rs +++ b/crates/tinyflows-drivers/src/store/import.rs @@ -17,7 +17,7 @@ use tinyflows::store::types::{ RunRecord, WorkflowError, WorkflowNote, WorkflowProposal, WorkflowRecord, WorkflowRevision, }; use tinyflows::store::{prune_notes, validate_graph}; -use tinystoragedrivers_core::{DocumentStore, Precondition}; +use tinyflows_storage::{DocumentStore, Precondition}; use super::definitions::{is_race, to_doc as definition_doc}; use super::history::to_doc as revision_doc; diff --git a/crates/tinyflows-drivers/src/store/import_tests.rs b/crates/tinyflows-drivers/src/store/import_tests.rs index a3ab65f0..a04e8ff1 100644 --- a/crates/tinyflows-drivers/src/store/import_tests.rs +++ b/crates/tinyflows-drivers/src/store/import_tests.rs @@ -4,7 +4,7 @@ use tinyflows::store::types::{ WorkflowRevision, }; use tinyflows::store::{MAX_NOTES, MAX_REVISIONS, WorkflowStore, mint_note_id, new_run_record}; -use tinystoragedrivers_core::MemoryStorage; +use tinyflows_storage::MemoryStorage; use crate::store::test_support::{record, store, store_over}; diff --git a/crates/tinyflows-drivers/src/store/locks.rs b/crates/tinyflows-drivers/src/store/locks.rs index 0e7fb723..24fd77c8 100644 --- a/crates/tinyflows-drivers/src/store/locks.rs +++ b/crates/tinyflows-drivers/src/store/locks.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use tinyflows::store::ProposalDecisionGuard; use tinyflows::store::types::WorkflowError; -use tinystoragedrivers_core::{Key, Leases}; +use tinyflows_storage::{Key, Leases}; use super::{DriverWorkflowStore, Inner, doc_id, storage}; diff --git a/crates/tinyflows-drivers/src/store/locks_tests.rs b/crates/tinyflows-drivers/src/store/locks_tests.rs index a072f23b..44754eb2 100644 --- a/crates/tinyflows-drivers/src/store/locks_tests.rs +++ b/crates/tinyflows-drivers/src/store/locks_tests.rs @@ -6,7 +6,7 @@ use std::time::Duration; use serde_json::json; use tinyflows::store::WorkflowStore; -use tinystoragedrivers_core::{Key, MemoryStorage, Precondition, primitives}; +use tinyflows_storage::{Key, MemoryStorage, Precondition, primitives}; use super::DriverWorkflowStore; use super::locks::{LEASE_OWNER, lease_key}; diff --git a/crates/tinyflows-drivers/src/store/migrate.rs b/crates/tinyflows-drivers/src/store/migrate.rs index 48540712..5b7c5c79 100644 --- a/crates/tinyflows-drivers/src/store/migrate.rs +++ b/crates/tinyflows-drivers/src/store/migrate.rs @@ -16,7 +16,7 @@ //! lost. use serde_json::{Map, Value, json}; -use tinystoragedrivers_core::{Key, Schema, ScopeClass, SourceRecord, Step, TargetRecord}; +use tinyflows_storage::{Key, Schema, ScopeClass, SourceRecord, Step, TargetRecord}; use super::codec::to_stored; use super::{ diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index deca7725..b61b9f7f 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use serde_json::{Value, json}; use tinyflows::store::WorkflowStore; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ DocumentStore, DocumentStoreExt, MemoryStorage, MigrateOptions, Precondition, Query, Registry, Scope, ScopeClass, ScopedStorage, StorageBackend, }; @@ -111,7 +111,7 @@ async fn a_hashed_v1_id_is_re_keyed_from_the_record_id() { migrate(&storage).await; - let key = tinystoragedrivers_core::Key::of([long.as_str()]).unwrap(); + let key = tinyflows_storage::Key::of([long.as_str()]).unwrap(); assert!( docs(&storage) .get("flows.workflows", key.as_str()) diff --git a/crates/tinyflows-drivers/src/store/mod.rs b/crates/tinyflows-drivers/src/store/mod.rs index 9294a859..47b54993 100644 --- a/crates/tinyflows-drivers/src/store/mod.rs +++ b/crates/tinyflows-drivers/src/store/mod.rs @@ -64,7 +64,7 @@ use std::sync::{Arc, OnceLock}; use tinyflows::store::types::WorkflowError; use tinyflows::store::{EnginePolicy, HostPolicy, MAX_NOTES}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Blocking, CollectionSpec, DocumentStore, IndexSpec, Key, Precondition, StorageError, Version, WriteOptions, primitives, }; diff --git a/crates/tinyflows-drivers/src/store/mod_tests.rs b/crates/tinyflows-drivers/src/store/mod_tests.rs index 633dd19e..f05c70c6 100644 --- a/crates/tinyflows-drivers/src/store/mod_tests.rs +++ b/crates/tinyflows-drivers/src/store/mod_tests.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use serde_json::json; use tinyflows::store::types::{WorkflowDefaults, WorkflowError, WorkflowRecord}; use tinyflows::store::{HostPolicy, WorkflowStore, parse_workflow, require}; -use tinystoragedrivers_core::{MemoryStorage, Precondition}; +use tinyflows_storage::{MemoryStorage, Precondition}; use super::test_support::{Faulty, block_on, docs_in, record, store, valid_document}; use super::*; diff --git a/crates/tinyflows-drivers/src/store/notes.rs b/crates/tinyflows-drivers/src/store/notes.rs index 6e8b0ff5..cc1ec7aa 100644 --- a/crates/tinyflows-drivers/src/store/notes.rs +++ b/crates/tinyflows-drivers/src/store/notes.rs @@ -7,7 +7,7 @@ use serde_json::{Value, json}; use tinyflows::store::prune_notes; use tinyflows::store::types::{WorkflowError, WorkflowNote}; -use tinystoragedrivers_core::{Key, Precondition, Versioned}; +use tinyflows_storage::{Key, Precondition, Versioned}; use super::codec::{journal_body, journal_from_body}; use super::definitions::is_race; diff --git a/crates/tinyflows-drivers/src/store/notes_tests.rs b/crates/tinyflows-drivers/src/store/notes_tests.rs index 06b5d215..aa109f44 100644 --- a/crates/tinyflows-drivers/src/store/notes_tests.rs +++ b/crates/tinyflows-drivers/src/store/notes_tests.rs @@ -5,7 +5,7 @@ use serde_json::json; use tinyflows::store::types::{NoteKind, NoteSource, WorkflowError, WorkflowNote}; use tinyflows::store::{MAX_NOTES, WorkflowStore, current_notes, mint_note_id}; -use tinystoragedrivers_core::{DocumentStoreExt, MemoryStorage, Precondition, Query}; +use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Precondition, Query}; use super::test_support::{block_on, docs_in, store, store_over}; use super::{DriverWorkflowStore, NOTES, QUARANTINE}; diff --git a/crates/tinyflows-drivers/src/store/proposals.rs b/crates/tinyflows-drivers/src/store/proposals.rs index ee864e23..dae02f6e 100644 --- a/crates/tinyflows-drivers/src/store/proposals.rs +++ b/crates/tinyflows-drivers/src/store/proposals.rs @@ -3,7 +3,7 @@ use serde_json::Value; use tinyflows::store::types::{WorkflowError, WorkflowProposal, fingerprint}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capability, DocumentStoreExt, ExpectedDocument, Filter, Precondition, Query, ReadCheck, Sort, Versioned, WriteOp, }; diff --git a/crates/tinyflows-drivers/src/store/proposals_tests.rs b/crates/tinyflows-drivers/src/store/proposals_tests.rs index f99c09c5..73a5e6dd 100644 --- a/crates/tinyflows-drivers/src/store/proposals_tests.rs +++ b/crates/tinyflows-drivers/src/store/proposals_tests.rs @@ -9,7 +9,7 @@ use std::sync::atomic::Ordering; use serde_json::json; use tinyflows::store::types::{ProposalStatus, WorkflowError, WorkflowProposal, fingerprint}; use tinyflows::store::{WorkflowStore, mint_proposal_id, require_proposal}; -use tinystoragedrivers_core::{DocumentStoreExt, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Precondition}; use super::test_support::{Faulty, block_on, docs_in, record, store, store_over}; use super::{DEFINITIONS, DriverWorkflowStore, PROPOSALS}; @@ -279,7 +279,7 @@ fn an_edit_that_lands_between_the_check_and_the_write_refuses_the_proposal() { "the graph moved, so the observation is stale" ); assert!( - block_on(docs.query_all(PROPOSALS, &tinystoragedrivers_core::Query::all())) + block_on(docs.query_all(PROPOSALS, &tinyflows_storage::Query::all())) .unwrap() .is_empty() ); diff --git a/crates/tinyflows-drivers/src/store/runs.rs b/crates/tinyflows-drivers/src/store/runs.rs index 9aa457c6..84d2f45f 100644 --- a/crates/tinyflows-drivers/src/store/runs.rs +++ b/crates/tinyflows-drivers/src/store/runs.rs @@ -2,7 +2,7 @@ use serde_json::Value; use tinyflows::store::types::{RunRecord, WorkflowError}; -use tinystoragedrivers_core::{DocumentStoreExt, Filter, Precondition, Query, Sort, Versioned}; +use tinyflows_storage::{DocumentStoreExt, Filter, Precondition, Query, Sort, Versioned}; use super::codec::{run_body, run_from_body}; use super::{Inner, RUNS, doc_id, put_doc, storage}; diff --git a/crates/tinyflows-drivers/src/store/runs_tests.rs b/crates/tinyflows-drivers/src/store/runs_tests.rs index cd357f90..2d019a70 100644 --- a/crates/tinyflows-drivers/src/store/runs_tests.rs +++ b/crates/tinyflows-drivers/src/store/runs_tests.rs @@ -126,7 +126,7 @@ fn an_executor_and_a_cancel_request_survive_a_write_and_read() { #[test] fn an_unreadable_run_document_is_skipped_by_listings_and_named_by_a_direct_read() { use serde_json::json; - use tinystoragedrivers_core::{MemoryStorage, Precondition}; + use tinyflows_storage::{MemoryStorage, Precondition}; let storage = MemoryStorage::new(); let docs = super::test_support::docs_in(&storage, "local"); let store = super::DriverWorkflowStore::new(docs.clone()); diff --git a/crates/tinyflows-drivers/src/store/test_support.rs b/crates/tinyflows-drivers/src/store/test_support.rs index 28ebc67e..a129b318 100644 --- a/crates/tinyflows-drivers/src/store/test_support.rs +++ b/crates/tinyflows-drivers/src/store/test_support.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use serde_json::json; use tinyflows::store::parse_workflow; use tinyflows::store::types::WorkflowRecord; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; use super::DriverWorkflowStore; @@ -99,23 +99,23 @@ impl Faulty { } } -#[tinystoragedrivers_core::async_trait] +#[tinyflows_storage::async_trait] impl DocumentStore for Faulty { - fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + fn capabilities(&self) -> tinyflows_storage::Capabilities { if self.transactions { self.inner.capabilities() } else { self.inner .capabilities() - .without(tinystoragedrivers_core::Capability::Transactions) - .without(tinystoragedrivers_core::Capability::ValidatedBatch) + .without(tinyflows_storage::Capability::Transactions) + .without(tinyflows_storage::Capability::ValidatedBatch) } } async fn ensure_collection( &self, - spec: &tinystoragedrivers_core::CollectionSpec, - ) -> tinystoragedrivers_core::Result<()> { + spec: &tinyflows_storage::CollectionSpec, + ) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -123,8 +123,8 @@ impl DocumentStore for Faulty { &self, collection: &str, id: &str, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { self.inner.get(collection, id).await } @@ -134,11 +134,11 @@ impl DocumentStore for Faulty { collection: &str, id: &str, doc: serde_json::Value, - precondition: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { if collection == self.collection && self.fail_puts.load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::backend( + return Err(tinyflows_storage::StorageError::backend( "injected write failure", )); } @@ -150,12 +150,12 @@ impl DocumentStore for Faulty { collection: &str, id: &str, doc: serde_json::Value, - precondition: tinystoragedrivers_core::Precondition, - options: tinystoragedrivers_core::WriteOptions, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + options: tinyflows_storage::WriteOptions, + ) -> tinyflows_storage::Result { if collection == self.collection && self.fail_puts.load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::backend( + return Err(tinyflows_storage::StorageError::backend( "injected write failure", )); } @@ -168,17 +168,17 @@ impl DocumentStore for Faulty { &self, collection: &str, id: &str, - precondition: tinystoragedrivers_core::Precondition, - ) -> tinystoragedrivers_core::Result { + precondition: tinyflows_storage::Precondition, + ) -> tinyflows_storage::Result { self.inner.delete(collection, id, precondition).await } async fn query( &self, collection: &str, - query: &tinystoragedrivers_core::Query, - ) -> tinystoragedrivers_core::Result< - tinystoragedrivers_core::Page>, + query: &tinyflows_storage::Query, + ) -> tinyflows_storage::Result< + tinyflows_storage::Page>, > { self.inner.query(collection, query).await } @@ -186,41 +186,41 @@ impl DocumentStore for Faulty { async fn count( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + filter: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } async fn delete_where( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - ) -> tinystoragedrivers_core::Result { + filter: &tinyflows_storage::Filter, + ) -> tinyflows_storage::Result { self.inner.delete_where(collection, filter).await } async fn claim( &self, collection: &str, - filter: &tinystoragedrivers_core::Filter, - sort: &[tinystoragedrivers_core::Sort], + filter: &tinyflows_storage::Filter, + sort: &[tinyflows_storage::Sort], patch: &serde_json::Value, - ) -> tinystoragedrivers_core::Result< - Option>, + ) -> tinyflows_storage::Result< + Option>, > { self.inner.claim(collection, filter, sort, patch).await } - async fn drop_collection(&self, collection: &str) -> tinystoragedrivers_core::Result<()> { + async fn drop_collection(&self, collection: &str) -> tinyflows_storage::Result<()> { self.inner.drop_collection(collection).await } async fn atomic_batch( &self, - ops: Vec, - ) -> tinystoragedrivers_core::Result> { + ops: Vec, + ) -> tinyflows_storage::Result> { if self.fail_batches.load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::backend( + return Err(tinyflows_storage::StorageError::backend( "injected batch failure", )); } @@ -229,11 +229,11 @@ impl DocumentStore for Faulty { async fn atomic_batch_checked( &self, - checks: Vec, - ops: Vec, - ) -> tinystoragedrivers_core::Result> { + checks: Vec, + ops: Vec, + ) -> tinyflows_storage::Result> { if self.fail_batches.load(std::sync::atomic::Ordering::SeqCst) { - return Err(tinystoragedrivers_core::StorageError::backend( + return Err(tinyflows_storage::StorageError::backend( "injected batch failure", )); } diff --git a/crates/tinyflows-module/Cargo.toml b/crates/tinyflows-module/Cargo.toml index 33b2500a..b413eb06 100644 --- a/crates/tinyflows-module/Cargo.toml +++ b/crates/tinyflows-module/Cargo.toml @@ -14,8 +14,10 @@ crate-type = ["rlib", "cdylib"] [dependencies] tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["catalog", "engine", "schedule"] } tinyflows-catalog = { path = "../tinyflows-catalog" } -tinyflows-import = { path = "../tinyflows-import", default-features = false, features = ["flow-write"] } -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinyflows-storage = { path = "../tinyflows-storage" } +tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } tinyflows-schedule = { path = "../tinyflows-schedule" } anyhow = "1" chrono.workspace = true @@ -41,3 +43,4 @@ static-link = [] [dev-dependencies] tempfile = "3" tokio = { workspace = true, features = ["test-util"] } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } diff --git a/crates/tinyflows-module/src/caps.rs b/crates/tinyflows-module/src/caps.rs index 1ec4a306..66412454 100644 --- a/crates/tinyflows-module/src/caps.rs +++ b/crates/tinyflows-module/src/caps.rs @@ -21,7 +21,7 @@ use tinyflows_bus::{ runs::{self, CapsHost, caps as members}, }; use tinyflows_drivers::catalog::{FlowCatalogDocuments, FlowStateDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; #[path = "caps/task_executor.rs"] mod task_executor; diff --git a/crates/tinyflows-module/src/caps_tests/scope_tests.rs b/crates/tinyflows-module/src/caps_tests/scope_tests.rs index 1280a895..c0ecaea6 100644 --- a/crates/tinyflows-module/src/caps_tests/scope_tests.rs +++ b/crates/tinyflows-module/src/caps_tests/scope_tests.rs @@ -5,7 +5,7 @@ use serde_json::{Value, json}; use std::sync::Arc; use tinyflows_bus::catalog::{self, CatalogStorage}; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; struct DocumentsHost(Arc); @@ -72,7 +72,7 @@ fn decode(value: &Value) -> tinybus::Result { serde_json::from_value(value.clone()).map_err(tinybus::Error::from) } -fn reply(result: tinystoragedrivers_core::Result) -> Value { +fn reply(result: tinyflows_storage::Result) -> Value { match result { Ok(value) => json!({"value": serde_json::to_value(value).unwrap()}), Err(error) => json!({"error":{ diff --git a/crates/tinyflows-module/src/catalog_documents.rs b/crates/tinyflows-module/src/catalog_documents.rs index e151258b..f6dc86b3 100644 --- a/crates/tinyflows-module/src/catalog_documents.rs +++ b/crates/tinyflows-module/src/catalog_documents.rs @@ -9,7 +9,7 @@ use tinyflows_bus::catalog::{ DocDeleteRequest, DocDeleteWhereRequest, DocEnsureRequest, DocGetRequest, DocPutRequest, DocQueryRequest, STORAGE_INTERFACE, storage, }; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capabilities, Capability, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Result, Sort, StorageError, Version, Versioned, WriteOp, WriteOptions, WriteResult, }; @@ -66,7 +66,7 @@ fn storage_error(error: CatalogStorageFailure) -> StorageError { "not_found" => StorageError::not_found(error.message), "invalid_input" => StorageError::invalid_input(error.message), "serialization" => StorageError::new( - tinystoragedrivers_core::ErrorKind::Serialization, + tinyflows_storage::ErrorKind::Serialization, error.message, ), "unavailable" => StorageError::unavailable(error.message), diff --git a/crates/tinyflows-module/src/catalog_extended_operations_tests.rs b/crates/tinyflows-module/src/catalog_extended_operations_tests.rs index 311f0786..448c4e6b 100644 --- a/crates/tinyflows-module/src/catalog_extended_operations_tests.rs +++ b/crates/tinyflows-module/src/catalog_extended_operations_tests.rs @@ -8,7 +8,7 @@ use tinyflows_bus::catalog::{ SuggestionStatus, UpdateDraftRequest, UpsertSuggestionsRequest, }; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; fn suggestion(id: &str) -> FlowSuggestion { FlowSuggestion { diff --git a/crates/tinyflows-module/src/catalog_extended_ops.rs b/crates/tinyflows-module/src/catalog_extended_ops.rs index 95227603..af490c28 100644 --- a/crates/tinyflows-module/src/catalog_extended_ops.rs +++ b/crates/tinyflows-module/src/catalog_extended_ops.rs @@ -17,7 +17,7 @@ use tinyflows_catalog::{ SuggestionStatus as StoredSuggestionStatus, }; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; async fn store( connection: &Connection, diff --git a/crates/tinyflows-module/src/catalog_legacy_import.rs b/crates/tinyflows-module/src/catalog_legacy_import.rs index 4b3b2fad..d3a98c4a 100644 --- a/crates/tinyflows-module/src/catalog_legacy_import.rs +++ b/crates/tinyflows-module/src/catalog_legacy_import.rs @@ -6,7 +6,7 @@ use serde_json::Value; use tinybus::{Connection, Error}; use tinyflows_bus::catalog::{ImportLegacyBatchRequest, ImportLegacyBatchResponse}; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; fn failure(error: impl std::fmt::Display) -> Error { Error::failed(error.to_string()) diff --git a/crates/tinyflows-module/src/catalog_migrations.rs b/crates/tinyflows-module/src/catalog_migrations.rs index b677d511..447e9b8e 100644 --- a/crates/tinyflows-module/src/catalog_migrations.rs +++ b/crates/tinyflows-module/src/catalog_migrations.rs @@ -6,7 +6,7 @@ use tinybus::Connection; use tinyflows_bus::catalog::CatalogStorage; use tinyflows_drivers::catalog::FlowCatalogDocuments; use tinyflows_drivers::{catalog_storage_schema, document_only_scope, import_legacy_kv}; -use tinystoragedrivers_core::{MigrateOptions, Registry, Scope}; +use tinyflows_storage::{MigrateOptions, Registry, Scope}; pub(super) async fn migrate(connection: Connection, storage: CatalogStorage) -> Result<()> { let mut documents = diff --git a/crates/tinyflows-module/src/catalog_service.rs b/crates/tinyflows-module/src/catalog_service.rs index 6c686c96..2e5d69c4 100644 --- a/crates/tinyflows-module/src/catalog_service.rs +++ b/crates/tinyflows-module/src/catalog_service.rs @@ -16,7 +16,7 @@ use tinyflows_bus::catalog::{ }; use tinyflows_catalog::Flow as StoredFlow; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; struct CatalogService { connection: Connection, diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index 46c9fcf3..06b64ecd 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -9,7 +9,7 @@ use tinyflows_bus::catalog::{ self, CallbackAddress, CatalogStorage, GetRevisionRequest, ListRevisionsRequest, PutWorkflowRequest, PutWorkflowResponse, }; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ CollectionSpec, DocumentStore, Key, MemoryStorage, Precondition, Scope, StorageBackend, WriteOptions, }; @@ -48,7 +48,7 @@ impl HostDocuments { && self.fail_revision_put.swap(false, Ordering::SeqCst) { return Ok(reply(Err::<(), _>( - tinystoragedrivers_core::StorageError::unavailable( + tinyflows_storage::StorageError::unavailable( "injected revision migration failure", ), ))); @@ -75,7 +75,7 @@ impl HostDocuments { async fn doc_delete(&self, request: Value) -> tinybus::Result { let result = if self.fail_draft_deletes && text(&request, "collection") == "flows.drafts" { - Err(tinystoragedrivers_core::StorageError::unavailable( + Err(tinyflows_storage::StorageError::unavailable( "injected draft delete failure", )) } else { @@ -115,7 +115,7 @@ fn decode(value: &Value) -> tinybus::Result { serde_json::from_value(value.clone()).map_err(tinybus::Error::from) } -fn reply(value: tinystoragedrivers_core::Result) -> Value { +fn reply(value: tinyflows_storage::Result) -> Value { match value { Ok(value) => json!({"value":serde_json::to_value(value).unwrap()}), Err(error) => { diff --git a/crates/tinyflows-module/src/run_manager/cancellation.rs b/crates/tinyflows-module/src/run_manager/cancellation.rs index 6234bd02..bc56e623 100644 --- a/crates/tinyflows-module/src/run_manager/cancellation.rs +++ b/crates/tinyflows-module/src/run_manager/cancellation.rs @@ -8,7 +8,7 @@ use tinybus::Connection; use tinyflows::graph::checkpoint::Checkpointer as _; use tinyflows_bus::runs::{CancelRunResponse, RunControlRequest, RunEvent}; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; use super::dedup_settlement::{DedupSettlementScope, from_snapshot}; use super::events::EventWriter; diff --git a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs index 2e1d3de2..a8b40b27 100644 --- a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs +++ b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs @@ -5,7 +5,7 @@ use std::task::Poll; use tinyflows::caps::StateStore as _; use tinyflows_bus::catalog::{CallbackAddress, CatalogStorage}; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; fn storage(path: &str) -> CatalogStorage { CatalogStorage { diff --git a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs index 95193afa..9230cc9e 100644 --- a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs +++ b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs @@ -8,7 +8,7 @@ use tinybus::Connection; use tinyflows_bus::catalog::{self, CatalogStorage}; use tinyflows_bus::runs::RunControlRequest; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capabilities, CollectionSpec, DocumentStore, Filter, MemoryStorage, Page, Precondition, Query, Scope, Sort, StorageBackend, StorageError, Version, Versioned, WriteOptions, }; @@ -175,7 +175,7 @@ impl DocumentStore for FaultingDocuments { async fn ensure_collection( &self, spec: &CollectionSpec, - ) -> tinystoragedrivers_core::Result<()> { + ) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -183,7 +183,7 @@ impl DocumentStore for FaultingDocuments { &self, collection: &str, id: &str, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.get(collection, id).await } @@ -193,7 +193,7 @@ impl DocumentStore for FaultingDocuments { id: &str, doc: Value, precondition: Precondition, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.put_with(collection, id, doc, precondition, WriteOptions::default()) .await } @@ -205,7 +205,7 @@ impl DocumentStore for FaultingDocuments { doc: Value, precondition: Precondition, options: WriteOptions, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { if Self::outcome_event(collection, &doc) { match self.mode { FaultMode::LimitOutcomeDocument(max_bytes) @@ -315,7 +315,7 @@ impl DocumentStore for FaultingDocuments { collection: &str, id: &str, precondition: Precondition, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.delete(collection, id, precondition).await } @@ -323,7 +323,7 @@ impl DocumentStore for FaultingDocuments { &self, collection: &str, query: &Query, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.query(collection, query).await } @@ -331,7 +331,7 @@ impl DocumentStore for FaultingDocuments { &self, collection: &str, filter: &Filter, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } @@ -339,7 +339,7 @@ impl DocumentStore for FaultingDocuments { &self, collection: &str, filter: &Filter, - ) -> tinystoragedrivers_core::Result { + ) -> tinyflows_storage::Result { self.inner.delete_where(collection, filter).await } @@ -349,11 +349,11 @@ impl DocumentStore for FaultingDocuments { filter: &Filter, sort: &[Sort], patch: &Value, - ) -> tinystoragedrivers_core::Result>> { + ) -> tinyflows_storage::Result>> { self.inner.claim(collection, filter, sort, patch).await } - async fn drop_collection(&self, collection: &str) -> tinystoragedrivers_core::Result<()> { + async fn drop_collection(&self, collection: &str) -> tinyflows_storage::Result<()> { self.inner.drop_collection(collection).await } } @@ -379,7 +379,7 @@ impl StorageReader { } } -fn reply(result: tinystoragedrivers_core::Result) -> Value { +fn reply(result: tinyflows_storage::Result) -> Value { match result { Ok(value) => json!({"value":serde_json::to_value(value).unwrap()}), Err(error) => json!({"error":{ diff --git a/crates/tinyflows-module/src/run_manager/preparation.rs b/crates/tinyflows-module/src/run_manager/preparation.rs index 5ef579b3..9859d645 100644 --- a/crates/tinyflows-module/src/run_manager/preparation.rs +++ b/crates/tinyflows-module/src/run_manager/preparation.rs @@ -14,7 +14,7 @@ use tinyflows::{ }; use tinyflows_bus::{catalog::CatalogStorage, runs::StartRun}; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; use super::super::catalog_documents::Documents; diff --git a/crates/tinyflows-module/src/run_manager/resume.rs b/crates/tinyflows-module/src/run_manager/resume.rs index 8738e02a..46e68972 100644 --- a/crates/tinyflows-module/src/run_manager/resume.rs +++ b/crates/tinyflows-module/src/run_manager/resume.rs @@ -8,7 +8,7 @@ use tinyflows::compiler; use tinyflows::engine::Checkpointer as _; use tinyflows_bus::runs::{FlowRunTrigger, ResumeRun, ResumeRunResponse, RunContext, RunEvent}; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; use super::actor::ActorRunRequest; use super::preparation::Prepared; diff --git a/crates/tinyflows-module/src/run_manager/sweeps.rs b/crates/tinyflows-module/src/run_manager/sweeps.rs index 055de321..225994c5 100644 --- a/crates/tinyflows-module/src/run_manager/sweeps.rs +++ b/crates/tinyflows-module/src/run_manager/sweeps.rs @@ -9,7 +9,7 @@ use tinyflows_bus::runs::{ BootSweep, BootSweepResponse, ExpireParked, ExpireParkedResponse, RunEvent, }; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; use super::dedup_settlement::{DedupSettlementScope, from_snapshot}; use super::preparation::catalog; diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index 3bc0e1e9..15774f34 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -12,7 +12,7 @@ use tinyflows_bus::runs::{ }; use tinyflows_catalog::FlowRunStep; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; use tokio::sync::oneshot; #[tokio::test] diff --git a/crates/tinyflows-module/src/run_manager_tests.rs b/crates/tinyflows-module/src/run_manager_tests.rs index 9cf72fcf..f4ee038c 100644 --- a/crates/tinyflows-module/src/run_manager_tests.rs +++ b/crates/tinyflows-module/src/run_manager_tests.rs @@ -1,7 +1,7 @@ use super::*; use tinyflows_bus::catalog::CallbackAddress; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; fn storage(owner: &str) -> tinyflows_bus::catalog::CatalogStorage { tinyflows_bus::catalog::CatalogStorage { diff --git a/crates/tinyflows-module/src/schedule_clock_tests.rs b/crates/tinyflows-module/src/schedule_clock_tests.rs index a6f59fbf..9cc196ca 100644 --- a/crates/tinyflows-module/src/schedule_clock_tests.rs +++ b/crates/tinyflows-module/src/schedule_clock_tests.rs @@ -17,7 +17,7 @@ async fn retry_sleep_uses_tokio_time_without_changing_the_calendar_instant() { #[tokio::test] async fn a_manual_pass_uses_the_supplied_time_for_due_jobs_and_run_records() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host @@ -161,7 +161,7 @@ fn a_host_clock_reads_system_time_until_the_host_sets_an_instant() { #[tokio::test] async fn ticking_at_a_later_instant_runs_a_future_job_without_waiting_for_it() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host diff --git a/crates/tinyflows-module/src/schedule_documents.rs b/crates/tinyflows-module/src/schedule_documents.rs index 515f5a10..415b59dd 100644 --- a/crates/tinyflows-module/src/schedule_documents.rs +++ b/crates/tinyflows-module/src/schedule_documents.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, de::DeserializeOwned}; use serde_json::{Value, json}; use tinybus::Connection; use tinyflows_bus::schedule::{CallbackAddress, DOCUMENT_INTERFACE, DOCUMENT_METHOD}; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capabilities, Capability, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Result, Sort, StorageError, Version, Versioned, WriteOp, WriteOptions, WriteResult, }; @@ -50,7 +50,7 @@ impl Failure { "not_found" => StorageError::not_found(self.message), "invalid_input" => StorageError::invalid_input(self.message), "serialization" => StorageError::new( - tinystoragedrivers_core::ErrorKind::Serialization, + tinyflows_storage::ErrorKind::Serialization, self.message, ), "unavailable" => StorageError::unavailable(self.message), diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index 372aef7e..296dea15 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -1,7 +1,7 @@ use super::*; use std::sync::Arc; -use tinystoragedrivers_core::{ +use tinyflows_storage::{ Capability, ErrorKind, MemoryStorage, Scope, StorageBackend, WriteOp, }; diff --git a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs index 9942af76..278a827c 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs @@ -6,7 +6,7 @@ async fn prepared_manual_worker_stop_wait_and_ack_are_idempotent_and_fenced() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, StoreConfig, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -278,7 +278,7 @@ async fn delivery_projection_error_releases_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; state diff --git a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs index d0583a27..facc9712 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs @@ -18,7 +18,7 @@ async fn claim_reply_lost_after_commit_is_reconciled_and_released_before_termina ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -134,7 +134,7 @@ async fn release_reply_loss_reports_cleanup_pending_until_retry_confirms_fence() ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -273,7 +273,7 @@ async fn unsupported_claim_backend_fails_before_recording_claim_intent() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -387,7 +387,7 @@ async fn terminal_commit_error_releases_queued_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() diff --git a/crates/tinyflows-module/src/schedule_members_tests.rs b/crates/tinyflows-module/src/schedule_members_tests.rs index 8d3ad457..be3f6232 100644 --- a/crates/tinyflows-module/src/schedule_members_tests.rs +++ b/crates/tinyflows-module/src/schedule_members_tests.rs @@ -203,9 +203,9 @@ async fn store_admission_cap_and_import_operations_are_served() { "/test/Documents".try_into().unwrap(), document_callbacks::Documents { docs: { - use tinystoragedrivers_core::StorageBackend; - tinystoragedrivers_core::MemoryStorage::new() - .for_scope(&tinystoragedrivers_core::Scope::local()) + use tinyflows_storage::StorageBackend; + tinyflows_storage::MemoryStorage::new() + .for_scope(&tinyflows_storage::Scope::local()) .unwrap() .documents() .clone() diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index 8841b88c..97869e4d 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -125,7 +125,7 @@ async fn separate_schedule_object_parses_delays_and_rejects_cross_tenant_deliver #[tokio::test] async fn run_job_now_is_a_typed_module_owned_admission_operation() { use tinyflows_bus::schedule::{ManualDelivery, RunJobNow, RunJobReply, StoreConfig}; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, state) = connections().await; let backend = MemoryStorage::new(); @@ -222,7 +222,7 @@ async fn run_job_now_returns_typed_refusal_when_a_scheduled_claim_is_active() { use tinyflows_bus::schedule::{ ManualDelivery, RunJobNow, RunJobRefusal, RunJobReply, StoreConfig, }; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; let (_server, host, _) = connections().await; let backend = MemoryStorage::new(); @@ -380,9 +380,9 @@ async fn module_job_storage_preserves_policy_updates_and_run_history() { "/test/Documents".try_into().unwrap(), document_callbacks::Documents { docs: { - use tinystoragedrivers_core::StorageBackend; - tinystoragedrivers_core::MemoryStorage::new() - .for_scope(&tinystoragedrivers_core::Scope::local()) + use tinyflows_storage::StorageBackend; + tinyflows_storage::MemoryStorage::new() + .for_scope(&tinyflows_storage::Scope::local()) .unwrap() .documents() .clone() @@ -425,7 +425,7 @@ mod queue_tests; #[tokio::test] async fn module_document_store_retries_policy_migration_conflicts_without_losing_seeded_jobs() { - use tinystoragedrivers_core::{MemoryStorage, Precondition, Scope, StorageBackend}; + use tinyflows_storage::{MemoryStorage, Precondition, Scope, StorageBackend}; let (_server, host, _) = connections().await; let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); let docs = scoped.documents().clone(); diff --git a/crates/tinyflows-module/src/schedule_store.rs b/crates/tinyflows-module/src/schedule_store.rs index 34e66f51..54fdb627 100644 --- a/crates/tinyflows-module/src/schedule_store.rs +++ b/crates/tinyflows-module/src/schedule_store.rs @@ -13,7 +13,7 @@ use tinyflows_schedule::{ CronJob, runtime::{Completion, RunRecord}, }; -use tinystoragedrivers_core::{MigrateOptions, Registry, Scope}; +use tinyflows_storage::{MigrateOptions, Registry, Scope}; /// The module's schedule store: `CronDocuments` over the host's document port. pub(super) struct Store(CronDocuments); diff --git a/crates/tinyflows-module/src/schedule_store_tests.rs b/crates/tinyflows-module/src/schedule_store_tests.rs index 9fc3cd52..d20570c9 100644 --- a/crates/tinyflows-module/src/schedule_store_tests.rs +++ b/crates/tinyflows-module/src/schedule_store_tests.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use std::time::Duration; use tinyflows_schedule::runtime::*; use tinyflows_schedule::{DeliveryStatus, Schedule}; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; use tokio::sync::Notify; struct Replacement(CronDocuments); diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index a274b7fc..39a9908c 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -3,7 +3,7 @@ //! These helpers contain record and migration behavior owned by TinyFlows; //! backend selection stays with the host. -pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; +pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, async_trait}; pub use tinystoragedrivers_types::*; mod key; pub use key::Key; diff --git a/crates/tinyflows-storage/src/primitives/counters.rs b/crates/tinyflows-storage/src/primitives/counters.rs index 7acba097..cf5f4df5 100644 --- a/crates/tinyflows-storage/src/primitives/counters.rs +++ b/crates/tinyflows-storage/src/primitives/counters.rs @@ -8,7 +8,7 @@ use super::COUNTERS; use super::repo::{Repo, RetryPolicy}; use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; -use tinystoragedrivers_types::Key; +use crate::Key; /// The largest value every driver can store (`i64::MAX`). const MAX_VALUE: u64 = i64::MAX as u64; diff --git a/crates/tinyflows-storage/src/primitives/kv.rs b/crates/tinyflows-storage/src/primitives/kv.rs index b26fdda5..1a3f7a9e 100644 --- a/crates/tinyflows-storage/src/primitives/kv.rs +++ b/crates/tinyflows-storage/src/primitives/kv.rs @@ -9,7 +9,8 @@ use serde_json::Value; use super::KV; use super::repo::Repo; use tinystoragedrivers_api::DocumentStore; -use tinystoragedrivers_types::{Filter, Key, Precondition, Query, Result, Sort, StorageError, Version}; +use tinystoragedrivers_types::{Filter, Precondition, Query, Result, Sort, StorageError, Version}; +use crate::Key; /// Entries per page while [`Kv::list`] walks a namespace. const PAGE: usize = 200; diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 5104d15d..3d5b33da 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -19,7 +19,7 @@ use super::LEASES; use super::repo::{Repo, RetryPolicy}; use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; -use tinystoragedrivers_types::Key; +use crate::Key; use crate::{Clock, system_clock}; /// The stored body: `{holder, epoch, expires_ms}` plus the optional From 665c4da20b9cea55e469a0d59b2f2e72e16b4749 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:16:58 +0300 Subject: [PATCH 09/64] chore(drivers): remove tinyflows-storage dependency The tinyflows-storage dependency has been removed from Cargo.toml as it is no longer required by the drivers crate. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tinyflows-drivers/Cargo.toml b/crates/tinyflows-drivers/Cargo.toml index 33aeb642..07eb1c56 100644 --- a/crates/tinyflows-drivers/Cargo.toml +++ b/crates/tinyflows-drivers/Cargo.toml @@ -31,7 +31,6 @@ catalog = [ "dep:anyhow", "dep:chrono", "dep:uuid", - "tinyflows-storage", ] # `schedule`: the cron job/run store (`schedule::CronDocuments`). It needs only # `tinyflows-schedule`, so a host that runs scheduled jobs but no flows can @@ -45,7 +44,6 @@ store = [ "engine", "tinyflows/store", "dep:uuid", - "tinyflows-storage", ] # The fixture-only writers (`force_run_status_for_test`, # `force_corrupt_graph_json_for_test`) that stage states no production path can From ae1a0dbc8b82efb13db360a8f6df2a4d11789ac7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:17:10 +0300 Subject: [PATCH 10/64] chore(deps): update dependencies and add new packages This updates several dependencies to newer versions, notably upgrading `windows-sys` across multiple packages. It also introduces several new packages related to storage drivers and bus communication. Auto-committed-on: dragonfly --- Cargo.lock | 83 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b4017b9..b8bf065b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,7 +428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1394,7 +1394,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1610,7 +1610,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1669,7 +1669,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1994,7 +1994,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2017,6 +2017,27 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "tinybus" +version = "0.1.5" +source = "git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7#d5b949be8ea56eb5a17289a09aa28e8c09bd88b7" +dependencies = [ + "async-trait", + "flate2", + "libc", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "thiserror", + "tokio", + "toml", + "tracing", + "ureq", + "zip", +] + [[package]] name = "tinybus" version = "0.1.5" @@ -2057,7 +2078,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "tinybus", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", "tokio", "tracing", ] @@ -2099,7 +2120,10 @@ dependencies = [ "thiserror", "tinyflows", "tinyflows-drivers", - "tinystoragedrivers-core", + "tinyflows-storage", + "tinystoragedrivers-api", + "tinystoragedrivers-testkit", + "tinystoragedrivers-types", "tokio", "tracing", ] @@ -2157,7 +2181,10 @@ dependencies = [ "tinyflows", "tinyflows-catalog", "tinyflows-schedule", - "tinystoragedrivers-core", + "tinyflows-storage", + "tinystoragedrivers-api", + "tinystoragedrivers-testkit", + "tinystoragedrivers-types", "tokio", "tracing", "uuid", @@ -2195,16 +2222,19 @@ dependencies = [ "serde", "serde_json", "tempfile", - "tinybus", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", "tinybus-module", "tinyflows", "tinyflows-bus", "tinyflows-catalog", "tinyflows-copilot", "tinyflows-drivers", - "tinyflows-import", "tinyflows-schedule", - "tinystoragedrivers-core", + "tinyflows-storage", + "tinystoragedrivers-api", + "tinystoragedrivers-busproxy", + "tinystoragedrivers-testkit", + "tinystoragedrivers-types", "tinytools", "tokio", "tracing", @@ -2272,6 +2302,35 @@ dependencies = [ "tinystoragedrivers-types", ] +[[package]] +name = "tinystoragedrivers-bus" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "serde", + "serde_json", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", + "tinystoragedrivers-types", +] + +[[package]] +name = "tinystoragedrivers-busproxy" +version = "1.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +dependencies = [ + "async-trait", + "bytes", + "futures-core", + "serde", + "serde_json", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", + "tinystoragedrivers-api", + "tinystoragedrivers-bus", + "tinystoragedrivers-types", + "tokio", + "zeroize", +] + [[package]] name = "tinystoragedrivers-connect" version = "1.0.0" @@ -2869,7 +2928,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From 9564fb524d69fee045d25fbd7d73745f09a8aaf6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:19:10 +0300 Subject: [PATCH 11/64] feat(storage): add blocking storage implementation This introduces a new blocking implementation for storage operations, allowing synchronous access to the storage layer. This is necessary to support environments where asynchronous operations are not available. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/blocking.rs | 159 +++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 crates/tinyflows-storage/src/blocking.rs diff --git a/crates/tinyflows-storage/src/blocking.rs b/crates/tinyflows-storage/src/blocking.rs new file mode 100644 index 00000000..58eb46fe --- /dev/null +++ b/crates/tinyflows-storage/src/blocking.rs @@ -0,0 +1,159 @@ +//! Calling the async ports from synchronous code. +//! +//! Some consumer traits that storage sits behind are still synchronous (a +//! workflow store, a keyring backend). [`Blocking`] runs a port's future to +//! completion on a runtime thread it owns, so a synchronous caller can use any +//! driver without knowing whether it is already inside a tokio runtime. +//! `block_in_place` would panic on a current-thread runtime, and +//! `Handle::block_on` from inside a runtime panics too; a dedicated thread has +//! neither problem. +//! +//! This is a bridge, not a destination: each synchronous consumer should +//! become async, at which point it calls the port directly. + +use std::fmt; +use std::future::Future; +use std::pin::Pin; +use std::sync::mpsc as std_mpsc; +use std::thread; + +use tokio::sync::mpsc; + +use tinystoragedrivers_types::{Result, StorageError}; + +type Job = Pin + Send>>; + +thread_local! { + static ON_BRIDGE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// A dedicated runtime thread that runs futures for synchronous callers. +/// +/// Many callers may share one `Blocking`: each submitted future runs as its own +/// task, so a slow call does not hold up the others. Dropping the last handle +/// stops the thread once in-flight work finishes. +/// +/// ``` +/// use tinystoragedrivers_core::{Blocking, MemoryStorage, Precondition, Scope, StorageBackend}; +/// use serde_json::json; +/// +/// let bridge = Blocking::new()?; +/// let docs = MemoryStorage::new().for_scope(&Scope::local())?.documents().clone(); +/// let version = bridge.run(async move { +/// docs.put("settings", "theme", json!({"dark": true}), Precondition::None).await +/// })??; +/// assert_eq!(version.0, 1); +/// # Ok::<(), tinystoragedrivers_core::StorageError>(()) +/// ``` +#[derive(Clone)] +pub struct Blocking { + jobs: mpsc::UnboundedSender, +} + +impl fmt::Debug for Blocking { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Blocking").finish_non_exhaustive() + } +} + +impl Blocking { + /// Start the runtime thread. + /// + /// # Errors + /// + /// [`ErrorKind::Backend`](crate::ErrorKind::Backend) when the runtime or + /// thread cannot be created. + pub fn new() -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(startup_error("cannot build the blocking bridge runtime"))?; + let (jobs, mut receiver) = mpsc::unbounded_channel::(); + thread::Builder::new() + .name("tinystoragedrivers-blocking".into()) + .spawn(move || { + ON_BRIDGE.with(|flag| flag.set(true)); + runtime.block_on(async move { + // Each call is its own task, so a slow one does not hold up + // the rest; the runtime drives them while this loop waits. + let mut running = tokio::task::JoinSet::new(); + while let Some(job) = receiver.recv().await { + running.spawn(job); + while running.try_join_next().is_some() {} + } + // Every handle is gone; let in-flight calls finish rather + // than cancelling them with the runtime. + while running.join_next().await.is_some() {} + }); + }) + .map_err(startup_error("cannot start the blocking bridge thread"))?; + Ok(Self { jobs }) + } + + /// Run `future` on the bridge thread and wait for its output. + /// + /// The calling thread blocks until the future completes. The future runs + /// on the bridge's own runtime, but it may still wait on tasks of the + /// caller's runtime: the MongoDB driver's connection pool, for one, runs + /// on the runtime that built the client. On a multi-thread runtime's + /// worker the wait goes through [`tokio::task::block_in_place`], so the + /// runtime keeps driving those tasks on another thread. From inside a + /// current-thread runtime nothing can, and a future that awaits a task + /// spawned on *that* runtime never completes. + /// + /// # Errors + /// + /// [`ErrorKind::Backend`](crate::ErrorKind::Backend) when called from the + /// bridge thread itself (which would deadlock), when the bridge has + /// stopped, or when the future panicked. + pub fn run(&self, future: F) -> Result + where + F: Future + Send + 'static, + T: Send + 'static, + { + if ON_BRIDGE.with(std::cell::Cell::get) { + return Err(StorageError::backend( + "a future running on the blocking bridge called back into it", + )); + } + let (reply, wait) = std_mpsc::sync_channel(1); + let job: Job = Box::pin(async move { + // The caller may have given up waiting; nothing to do then. + let _ = reply.send(future.await); + }); + self.jobs + .send(job) + .map_err(|_| StorageError::backend("the blocking bridge has stopped"))?; + let receive = || { + wait.recv().map_err(|_| { + StorageError::backend("the blocking bridge dropped the call (did it panic?)") + }) + }; + // On a multi-thread runtime's worker, hand the worker's other tasks to + // another thread while this one waits. A driver's future may depend on + // them: MongoDB's connection pool runs as a task on the runtime that + // built the client, and with this worker blocked (or the runtime's only + // one) the call would never complete. + if on_multi_thread_worker() { + tokio::task::block_in_place(receive) + } else { + receive() + } + } +} + +/// Whether the current thread runs inside a multi-thread tokio runtime, +/// where [`tokio::task::block_in_place`] is allowed. +fn on_multi_thread_worker() -> bool { + tokio::runtime::Handle::try_current() + .is_ok_and(|handle| handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread) +} + +/// Map an I/O failure while starting the bridge to a backend error. +fn startup_error(what: &'static str) -> impl FnOnce(std::io::Error) -> StorageError { + move |error| StorageError::backend(what).with_source(error) +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; From 441526525d953d379cb324e023a72e273ce9c85e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:21:08 +0300 Subject: [PATCH 12/64] feat(storage): expose schema and blocking modules This change exposes the schema and blocking modules from the storage crate. This allows users to interact with schema migration and blocking mechanisms directly from the library. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/lib.rs | 4 + crates/tinyflows-storage/src/schema.rs | 272 +++++++++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 crates/tinyflows-storage/src/schema.rs diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 39a9908c..89beb90d 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -6,7 +6,11 @@ pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, async_trait}; pub use tinystoragedrivers_types::*; mod key; +mod schema; pub use key::Key; +pub use schema::{MigrateOptions, MigrationReport, Migrations, Owner, Schema, ScopeClass, SourceRecord, Step, StepReport, TargetRecord}; +mod blocking; +pub use blocking::Blocking; pub use primitives::{Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy}; pub(crate) type Clock = std::sync::Arc u64 + Send + Sync>; diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs new file mode 100644 index 00000000..a83a5244 --- /dev/null +++ b/crates/tinyflows-storage/src/schema.rs @@ -0,0 +1,272 @@ +//! TinyFlows-owned collection declarations and resumable owner migrations. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; +use tinystoragedrivers_types::{ + CollectionSpec, Filter, Meta, Precondition, Query, Result, StorageError, Versioned, + WriteOptions, SCHEMA_FIELD, +}; + +use crate::Key; + +const PAGE_SIZE: usize = 100; + +/// Which product scope owns a declared collection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScopeClass { + /// Tenant or local user data. + Tenant, +} + +/// A validated TinyFlows storage owner name. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Owner(String); + +impl Owner { + /// Validate an owner name. + pub fn new(name: &str) -> Result { + let valid = (2..=24).contains(&name.len()) + && name.as_bytes().first().is_some_and(u8::is_ascii_lowercase) + && name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); + if !valid || name == "tsd" { + return Err(StorageError::invalid_input("invalid storage owner name")); + } + Ok(Self(name.to_owned())) + } + + /// Owner name. + #[must_use] + pub fn as_str(&self) -> &str { &self.0 } +} + +/// A source document passed to a migration step. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceRecord { + /// Existing source id. + pub id: String, + /// Existing document body. + pub body: Value, + /// Existing storage metadata. + pub meta: Meta, +} + +/// A document emitted by a migration step. +#[derive(Debug, Clone, PartialEq)] +pub struct TargetRecord { + /// Destination collection. + pub collection: String, + /// Destination id. + pub id: String, + /// Destination body. + pub body: Value, + /// Creation timestamp to preserve where available. + pub created_ms: Option, +} + +impl TargetRecord { + /// Create a migrated document. + #[must_use] + pub fn new(collection: impl Into, id: impl Into, body: Value) -> Self { + Self { collection: collection.into(), id: id.into(), body, created_ms: None } + } + + /// Preserve the source creation timestamp. + #[must_use] + pub fn created_ms(mut self, value: i64) -> Self { self.created_ms = Some(value); self } +} + +type StepFn = dyn Fn(Vec) -> Result> + Send + Sync; + +/// One document-collection migration step. +#[derive(Clone)] +pub struct Step { + /// Stable step id. + pub id: &'static str, + /// Destination collection. + pub item: &'static str, + /// Source schema version. + pub from: u32, + /// Destination schema version. + pub to: u32, + /// Legacy source collection, when different from `item`. + pub source: Option<&'static str>, + run: Arc, +} + +impl Step { + /// Define a document migration step. + #[must_use] + pub fn new( + id: &'static str, + item: &'static str, + from: u32, + to: u32, + run: impl Fn(Vec) -> Result> + Send + Sync + 'static, + ) -> Self { + Self { id, item, from, to, source: None, run: Arc::new(run) } + } + + /// Read from a legacy collection rather than the destination. + #[must_use] + pub fn source(mut self, name: &'static str) -> Self { self.source = Some(name); self } +} + +/// One owner’s collections and migrations, opened by that owner. +#[derive(Debug, Clone)] +pub struct Schema { + /// Storage owner for these collections. + pub owner: Owner, + /// Declared collections and versions. + pub items: Vec<(CollectionSpec, u32, ScopeClass)>, + /// Ordered migrations. + pub steps: Vec, +} + +impl Schema { + /// Start a declaration for `owner`. + #[must_use] + pub fn new(owner: Owner) -> Self { Self { owner, items: Vec::new(), steps: Vec::new() } } + + /// Declare a collection and schema version. + #[must_use] + pub fn collection(self, name: &str, version: u32, scope: ScopeClass) -> Self { + self.collection_with(version, CollectionSpec::new(name), scope) + } + + /// Declare a fully specified collection and schema version. + #[must_use] + pub fn collection_with(mut self, version: u32, spec: CollectionSpec, scope: ScopeClass) -> Self { + self.items.push((spec, version, scope)); + self + } + + /// Add an ordered collection migration. + #[must_use] + pub fn step(mut self, step: Step) -> Self { self.steps.push(step); self } + + /// Pure collection declarations for host bootstrap and documentation. + #[must_use] + pub fn collections(&self) -> Vec { + let mut specs = self.items.iter().map(|(spec, _, _)| spec.clone()).collect::>(); + specs.push(CollectionSpec::new(self.migrations_collection())); + specs + } + + fn migrations_collection(&self) -> String { format!("{}.migrations", self.owner.as_str()) } + + /// Ensure this owner’s collections, then migrate its data in place. + /// + /// A separate `.migrations` collection records one progress row per + /// step. Backends are supplied by the caller; this method never constructs + /// an engine or an in-memory fallback. + pub async fn migrate(&self, docs: &Arc, options: &MigrateOptions) -> Result { + if options.batch_size == 0 { return Err(StorageError::invalid_input("migration batch size must be at least 1")); } + for spec in self.collections() { docs.ensure_collection(&spec).await?; } + let migrations = Migrations { docs: Arc::clone(docs), collection: self.migrations_collection() }; + let mut report = MigrationReport::default(); + for step in &self.steps { + if migrations.done(self.owner.as_str(), step.id).await? { report.already_done += 1; continue; } + let progress_id = Key::of([self.owner.as_str(), step.id])?.as_str().to_owned(); + let progress = migrations.docs.get_as::(&migrations.collection, &progress_id).await?; + let mut query = Query::filter(Filter::all()).limit(options.batch_size); + if let Some(cursor) = progress.and_then(|p| p.doc.cursor) { + query = query.after(tinystoragedrivers_types::Cursor(cursor)); + } + let source = step.source.unwrap_or(step.item); + let mut written = 0_u64; + loop { + let page = docs.query(source, &query).await?; + if page.items.is_empty() { break; } + let last_cursor = page.next.clone(); + let inputs = page.items.iter().map(|row| SourceRecord { id: row.id.clone(), body: row.doc.clone(), meta: row.meta.clone() }).collect(); + let outputs = (step.run)(inputs)?; + let output_ids: std::collections::HashSet<(String, String)> = outputs.iter().map(|r| (r.collection.clone(), r.id.clone())).collect(); + for mut target in outputs { + if !target.body.is_object() { return Err(StorageError::invalid_input("migration output must be an object")); } + target.body[SCHEMA_FIELD] = step.to.into(); + let options = target.created_ms.map_or_else(WriteOptions::new, |ms| WriteOptions::new().created_ms(ms)); + match docs.put_with(&target.collection, &target.id, target.body.clone(), Precondition::Absent, options).await { + Ok(_) => written += 1, + Err(error) if error.kind() == tinystoragedrivers_types::ErrorKind::Conflict => { + let existing = docs.get(&target.collection, &target.id).await?; + if existing.is_none_or(|row| row.doc != target.body) { return Err(error); } + written += 1; + } + Err(error) => return Err(error), + } + } + if source != step.item { + for row in &page.items { + if output_ids.contains(&(step.item.to_owned(), row.id.clone())) { + docs.delete(source, &row.id, Precondition::Version(row.version)).await?; + } + } + } + let cursor = last_cursor.map(|cursor| cursor.0); + migrations.save(&progress_id, &Progress { owner: self.owner.as_str().to_owned(), step: step.id.to_owned(), cursor, done: false }).await?; + if let Some(next) = page.next { query = query.after(next); } else { break; } + } + migrations.save(&progress_id, &Progress { owner: self.owner.as_str().to_owned(), step: step.id.to_owned(), cursor: None, done: true }).await?; + report.ran.push(StepReport { owner: self.owner.as_str().to_owned(), item: step.item.to_owned(), step: step.id.to_owned(), written }); + } + Ok(report) + } +} + +/// Batch size for owner migration runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MigrateOptions { batch_size: usize } +impl MigrateOptions { + /// Defaults to 100 documents per batch. + #[must_use] + pub const fn new() -> Self { Self { batch_size: PAGE_SIZE } } + /// Set the number of source documents read per batch. + #[must_use] + pub const fn batch_size(mut self, batch_size: usize) -> Self { self.batch_size = batch_size; self } +} +impl Default for MigrateOptions { fn default() -> Self { Self::new() } } + +/// Summary of an owner migration run. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct MigrationReport { /// Steps completed by this invocation. + pub ran: Vec, /// Steps already marked complete. + pub already_done: usize } + +/// A completed migration step. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StepReport { /// Owner name. + pub owner: String, /// Collection being migrated. + pub item: String, /// Stable step identifier. + pub step: String, /// Number of destination records written. + pub written: u64 } + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Progress { owner: String, step: String, cursor: Option, done: bool } + +/// Per-owner migration completion markers. +#[derive(Debug, Clone)] +pub struct Migrations { docs: Arc, collection: String } +impl Migrations { + /// Bind markers to the caller-provided document store and private collection. + #[must_use] + pub fn new(docs: &Arc, owner: &str) -> Self { Self { docs: Arc::clone(docs), collection: format!("{owner}.migrations") } } + + /// Whether a step has completed. + pub async fn done(&self, owner: &str, step: &str) -> Result { + let id = Key::of([owner, step])?.as_str().to_owned(); + Ok(self.docs.get_as::(&self.collection, &id).await?.is_some_and(|row| row.doc.done)) + } + + /// Mark an external import step complete. + pub async fn mark_done(&self, owner: &str, step: &str) -> Result<()> { + let id = Key::of([owner, step])?.as_str().to_owned(); + self.save(&id, &Progress { owner: owner.to_owned(), step: step.to_owned(), cursor: None, done: true }).await + } + + async fn save(&self, id: &str, progress: &Progress) -> Result<()> { + self.docs.put_as(&self.collection, id, progress, Precondition::None).await.map(|_| ()) + } +} From 9268437433cf64c76b60edc2a4327fbecd8c433e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:21:29 +0300 Subject: [PATCH 13/64] refactor(storage): update storage API usage This refactoring updates how storage schemas and migrations are invoked across the module. It removes reliance on `document_only_scope` in favor of directly using the storage schema's migration methods, simplifying the migration logic. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/lib.rs | 2 +- crates/tinyflows-drivers/src/storage/mod.rs | 35 ++++++------------- .../src/catalog_migrations.rs | 13 +++---- crates/tinyflows-module/src/schedule_store.rs | 11 +++--- 4 files changed, 21 insertions(+), 40 deletions(-) diff --git a/crates/tinyflows-drivers/src/lib.rs b/crates/tinyflows-drivers/src/lib.rs index e81bdfa7..884c1aa8 100644 --- a/crates/tinyflows-drivers/src/lib.rs +++ b/crates/tinyflows-drivers/src/lib.rs @@ -61,6 +61,6 @@ pub use state_store::DriverStateStore; #[cfg(feature = "catalog")] pub use storage::catalog_storage_schema; -pub use storage::{KV_OWNER, document_only_scope, import_legacy_kv, storage_schema}; +pub use storage::{KV_OWNER, collections, import_legacy_kv, storage_schema}; #[cfg(feature = "schedule")] pub use storage::{cron_storage_schema, import_legacy_cron_counters}; diff --git a/crates/tinyflows-drivers/src/storage/mod.rs b/crates/tinyflows-drivers/src/storage/mod.rs index fa654802..754e9f64 100644 --- a/crates/tinyflows-drivers/src/storage/mod.rs +++ b/crates/tinyflows-drivers/src/storage/mod.rs @@ -22,11 +22,7 @@ mod kv_import; -use std::sync::Arc; - -use tinyflows_storage::{ - DocumentStore, MemoryStorage, Owner, Schema, Scope, ScopedStorage, StorageBackend, -}; +use tinyflows_storage::{CollectionSpec, Owner, Schema, ScopeClass}; pub use kv_import::{ CATALOG_NAMESPACE_PREFIX, KV_OWNER, STATE_NAMESPACE, catalog_namespace, import_legacy_kv, @@ -60,6 +56,16 @@ pub fn storage_schema() -> Schema { schema } +/// Collections owned by TinyFlows drivers for host bootstrap and docs. +#[must_use] +pub fn collections() -> Vec { + let mut specs = storage_schema().collections(); + #[cfg(feature = "schedule")] + specs.extend(cron_storage_schema().collections()); + specs.extend(tinyflows_storage::primitives::collection_specs()); + specs +} + /// The catalog-only portion of owner `flows` and its v1 -> v2 steps. /// /// Unlike [`storage_schema`], this excludes workflow-store schemas so the @@ -70,25 +76,6 @@ pub fn catalog_storage_schema() -> Schema { crate::catalog::extend_schema(flows_schema()) } -/// Assemble a scoped document-only migration view over a host callback. -/// -/// The stream and blob handles are inert memory ports because the TinyFlows -/// catalog and cron migration schemas contain document steps only. They are -/// never read or written by these migrations. -pub fn document_only_scope( - scope: Scope, - documents: Arc, -) -> tinyflows_storage::Result { - let memory = MemoryStorage::new().for_scope(&scope)?; - Ok(ScopedStorage::new( - scope, - "tinyflows-document-callback", - documents, - memory.streams().clone(), - memory.blobs().clone(), - )) -} - /// The `cron` owner's declarations and v1 -> v2 steps. #[cfg(feature = "schedule")] #[must_use] diff --git a/crates/tinyflows-module/src/catalog_migrations.rs b/crates/tinyflows-module/src/catalog_migrations.rs index 447e9b8e..ae89cedf 100644 --- a/crates/tinyflows-module/src/catalog_migrations.rs +++ b/crates/tinyflows-module/src/catalog_migrations.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use tinybus::Connection; use tinyflows_bus::catalog::CatalogStorage; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinyflows_drivers::{catalog_storage_schema, document_only_scope, import_legacy_kv}; -use tinyflows_storage::{MigrateOptions, Registry, Scope}; +use tinyflows_drivers::{catalog_storage_schema, import_legacy_kv}; +use tinyflows_storage::MigrateOptions; pub(super) async fn migrate(connection: Connection, storage: CatalogStorage) -> Result<()> { let mut documents = @@ -14,13 +14,10 @@ pub(super) async fn migrate(connection: Connection, storage: CatalogStorage) -> documents.require_record_metadata().await?; let _guard = super::migration_lock::catalog().await; let documents = Arc::new(documents); - let scoped = document_only_scope(Scope::local(), documents.clone())?; FlowCatalogDocuments::new(documents).ensure().await?; - let registry = Registry::compose([catalog_storage_schema()])?; - let report = registry - .migrate(&scoped, &MigrateOptions::default()) - .await?; - let imported_kv = import_legacy_kv(&scoped).await?; + let schema = catalog_storage_schema(); + let report = schema.migrate(&documents, &MigrateOptions::default()).await?; + let imported_kv = import_legacy_kv(&documents).await?; tracing::debug!( steps_run = report.ran.len(), already_done = report.already_done, diff --git a/crates/tinyflows-module/src/schedule_store.rs b/crates/tinyflows-module/src/schedule_store.rs index 54fdb627..10b4d445 100644 --- a/crates/tinyflows-module/src/schedule_store.rs +++ b/crates/tinyflows-module/src/schedule_store.rs @@ -13,7 +13,7 @@ use tinyflows_schedule::{ CronJob, runtime::{Completion, RunRecord}, }; -use tinyflows_storage::{MigrateOptions, Registry, Scope}; +use tinyflows_storage::MigrateOptions; /// The module's schedule store: `CronDocuments` over the host's document port. pub(super) struct Store(CronDocuments); @@ -44,16 +44,13 @@ impl Store { Arc::new(super::schedule_documents::Documents::new(connection, callback).await?); documents.require_record_metadata()?; let _guard = super::migration_lock::schedule().await; - let storage = tinyflows_drivers::document_only_scope(Scope::local(), documents.clone())?; CronDocuments::new(documents) .ensure() .await .map_err(anyhow::Error::msg)?; - let registry = Registry::compose([tinyflows_drivers::cron_storage_schema()])?; - let report = registry - .migrate(&storage, &MigrateOptions::default()) - .await?; - let counter_imported = tinyflows_drivers::import_legacy_cron_counters(&storage).await?; + let schema = tinyflows_drivers::cron_storage_schema(); + let report = schema.migrate(&documents, &MigrateOptions::default()).await?; + let counter_imported = tinyflows_drivers::import_legacy_cron_counters(&documents).await?; Ok(serde_json::json!({ "steps_run": report.ran.len(), "already_done": report.already_done, From 134c4e2e362e11f30b9a23aebded4a4e53c6610a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:21:56 +0300 Subject: [PATCH 14/64] chore: files changed crates/tinyflows-adaptive/src/drivers/migrate.rs,crates/tinyflows-drivers/src/s Auto-committed-on: dragonfly --- .../tinyflows-adaptive/src/drivers/migrate.rs | 11 ++++++----- .../tinyflows-drivers/src/schedule/migrate.rs | 11 ++++++----- .../src/storage/kv_import.rs | 19 ++++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index 82c8200f..d47651fc 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -20,8 +20,9 @@ //! visible (readers report it) instead of being lost. use serde_json::{Map, Value}; +use std::sync::Arc; use tinyflows_storage::{ - Cursor, Filter, ID_FIELD, Key, Migrations, Query, Result, Schema, ScopeClass, ScopedStorage, + Cursor, DocumentStore, Filter, ID_FIELD, Key, Migrations, Query, Result, Schema, ScopeClass, Sort, SourceRecord, Step, TargetRecord, primitives, }; @@ -224,13 +225,13 @@ fn workflow(source: &SourceRecord) -> Option { /// /// The first storage error; the import resumes on the next call because /// nothing is removed before the counters are raised. -pub async fn import_legacy_adaptive_counters(storage: &ScopedStorage) -> Result { - let migrations = Migrations::new(storage); +pub async fn import_legacy_adaptive_counters(docs: &Arc) -> Result { + let migrations = Migrations::new(docs, "adaptive"); if migrations.done(COUNTER_OWNER, COUNTERS_STEP).await? { return Ok(0); } - primitives::ensure(storage).await?; - let docs = storage.documents(); + primitives::ensure(docs).await?; + docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("adaptive.migrations")).await?; docs.ensure_collection(&tinyflows_storage::CollectionSpec::new(COUNTERS)) .await?; let counters = tinyflows_storage::Counters::new(docs); diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index 8434c030..9a9f8e32 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -21,8 +21,9 @@ //! moves it into `tsd.counters` once per scope after `Registry::migrate`. use serde_json::{Map, Value, json}; +use std::sync::Arc; use tinyflows_storage::{ - Counters, Migrations, Owner, Precondition, Result, Schema, ScopeClass, ScopedStorage, + Counters, DocumentStore, Migrations, Owner, Precondition, Result, Schema, ScopeClass, SourceRecord, Step, TargetRecord, }; @@ -248,13 +249,13 @@ fn counter_record(source: &SourceRecord) -> Option { /// # Errors /// /// The first storage error; the import resumes on the next call. -pub async fn import_legacy_cron_counters(storage: &ScopedStorage) -> Result { - let migrations = Migrations::new(storage); +pub async fn import_legacy_cron_counters(docs: &Arc) -> Result { + let migrations = Migrations::new(docs, OWNER); if migrations.done(OWNER, COUNTERS_STEP).await? { return Ok(false); } - tinyflows_storage::primitives::ensure(storage).await?; - let docs = storage.documents(); + tinyflows_storage::primitives::ensure(docs).await?; + docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("cron.migrations")).await?; docs.ensure_collection(&CronDocuments::collections().remove(2)) .await?; let mut carried = false; diff --git a/crates/tinyflows-drivers/src/storage/kv_import.rs b/crates/tinyflows-drivers/src/storage/kv_import.rs index 8873aa71..b5633dce 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import.rs @@ -12,9 +12,10 @@ //! is never overwritten by a stale v1 row. use serde_json::Value; +use std::sync::Arc; use tinyflows_storage::{ - CollectionSpec, Cursor, ErrorKind, Filter, ID_FIELD, Key, Migrations, Precondition, Query, - Result, ScopedStorage, Sort, primitives, + CollectionSpec, Cursor, DocumentStore, ErrorKind, Filter, ID_FIELD, Key, Migrations, + Precondition, Query, Result, Sort, primitives, }; /// The `tsd.kv` owner of every entry tinyflows stores. @@ -51,11 +52,10 @@ fn convert_state(body: &Value) -> Option<(String, String, Value)> { } async fn copy( - storage: &ScopedStorage, + docs: &Arc, legacy: &str, convert: fn(&Value) -> Option<(String, String, Value)>, ) -> Result { - let docs = storage.documents(); docs.ensure_collection(&CollectionSpec::new(legacy)).await?; let mut written = 0_u64; let mut cursor: Option = None; @@ -104,15 +104,16 @@ async fn copy( /// /// The first storage error; the import resumes on the next call because /// nothing is removed before its collection has been copied. -pub async fn import_legacy_kv(storage: &ScopedStorage) -> Result { - let migrations = Migrations::new(storage); +pub async fn import_legacy_kv(docs: &Arc) -> Result { + let migrations = Migrations::new(docs, KV_OWNER); if migrations.done(KV_OWNER, STEP).await? { tracing::debug!("[flows-storage] kv import already done"); return Ok(0); } - primitives::ensure(storage).await?; - let written = copy(storage, "flows_kv", convert_catalog).await? - + copy(storage, "flows_state", convert_state).await?; + primitives::ensure(docs).await?; + docs.ensure_collection(&CollectionSpec::new("flows.migrations")).await?; + let written = copy(docs, "flows_kv", convert_catalog).await? + + copy(docs, "flows_state", convert_state).await?; migrations.mark_done(KV_OWNER, STEP).await?; tracing::debug!(written, "[flows-storage] kv import complete"); Ok(written) From a367ebbbcc98e5e06dffb0811acf576727b3b8d6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:22:16 +0300 Subject: [PATCH 15/64] chore: files changed crates/tinyflows-adaptive/src/drivers/ledger_tests.rs,crates/tinyflows-adaptive Auto-committed-on: dragonfly --- .../tinyflows-adaptive/src/drivers/ledger_tests.rs | 2 +- .../tinyflows-adaptive/src/drivers/migrate_tests.rs | 12 ++++++------ crates/tinyflows-adaptive/src/drivers/vault_tests.rs | 2 +- crates/tinyflows-adaptive/src/storage_tests.rs | 2 +- .../src/catalog/definitions_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/migrate_tests.rs | 12 ++++++------ .../src/catalog/revisions_gap_tests.rs | 2 +- .../src/catalog/runs_recovery_lease_tests.rs | 2 +- .../src/catalog/runs_replace_tests.rs | 2 +- .../tinyflows-drivers/src/checkpoint_import_tests.rs | 2 +- .../tinyflows-drivers/src/checkpoint_layout_tests.rs | 2 +- crates/tinyflows-drivers/src/checkpoint_tests.rs | 4 ++-- .../src/schedule/claims_compaction_tests.rs | 2 +- .../src/schedule/claims_safety_tests.rs | 2 +- .../src/schedule/manual_claim_fence_tests.rs | 2 +- .../tinyflows-drivers/src/schedule/migrate_tests.rs | 10 +++++----- crates/tinyflows-drivers/src/schedule/mod_tests.rs | 2 +- crates/tinyflows-drivers/src/state_store_tests.rs | 4 ++-- .../tinyflows-drivers/src/storage/kv_import_tests.rs | 6 +++--- .../tinyflows-drivers/src/store/authoring_tests.rs | 2 +- crates/tinyflows-drivers/src/store/locks_tests.rs | 2 +- crates/tinyflows-drivers/src/store/migrate_tests.rs | 12 ++++++------ crates/tinyflows-drivers/src/store/mod_tests.rs | 2 +- crates/tinyflows-drivers/src/store/notes_tests.rs | 2 +- .../tinyflows-drivers/src/store/proposals_tests.rs | 2 +- crates/tinyflows-drivers/src/store/runs_tests.rs | 2 +- .../tinyflows-module/src/caps_tests/scope_tests.rs | 2 +- crates/tinyflows-module/src/catalog_service_tests.rs | 2 +- .../src/run_manager/dedup_settlement_tests.rs | 2 +- .../src/run_manager_actor_recovery_tests.rs | 2 +- crates/tinyflows-module/src/run_manager_tests.rs | 2 +- crates/tinyflows-module/src/schedule_clock_tests.rs | 4 ++-- .../tinyflows-module/src/schedule_documents_tests.rs | 2 +- ...schedule_manual_worker_lifecycle_service_tests.rs | 4 ++-- .../src/schedule_manual_worker_service_tests.rs | 8 ++++---- .../tinyflows-module/src/schedule_service_tests.rs | 6 +++--- crates/tinyflows-module/src/schedule_store_tests.rs | 2 +- crates/tinyflows-storage/src/lib.rs | 2 +- 38 files changed, 68 insertions(+), 68 deletions(-) diff --git a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs index e3d79ede..d554e206 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs @@ -3,7 +3,7 @@ use crate::drivers::key; use crate::execute::StepRecord; use crate::ledger::{Ledger, Score, conformance}; use tinyflows_storage::Counters; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index d625ffd5..d6331cbf 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use serde_json::{Value, json}; use tinyflows_storage::{ - Counters, DocumentStore, DocumentStoreExt, Key, MemoryStorage, MigrateOptions, Owner, - Precondition, Registry, Schema, Scope, ScopeClass, ScopedStorage, StorageBackend, + Counters, DocumentStore, DocumentStoreExt, Key, MigrateOptions, Owner, + Precondition, Registry, Schema, Scope, ScopeClass, Storage, StorageBackend, }; use super::*; @@ -11,11 +11,11 @@ use crate::drivers::{DriverLedger, DriverVault}; use crate::ledger::{Ledger, conformance}; use crate::workflows::Vault; -fn scoped() -> ScopedStorage { +fn scoped() -> Storage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() } -fn docs(storage: &ScopedStorage) -> Arc { +fn docs(storage: &Storage) -> Arc { Arc::clone(storage.documents()) } @@ -23,7 +23,7 @@ fn schema() -> Schema { extend_storage_schema(Schema::new(Owner::new("flows").unwrap())) } -async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) { +async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { storage .documents() .put(collection, id, body, Precondition::None) @@ -31,7 +31,7 @@ async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) .unwrap(); } -async fn migrate(storage: &ScopedStorage) { +async fn migrate(storage: &Storage) { Registry::compose([schema()]) .unwrap() .migrate(storage, &MigrateOptions::new()) diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index b9e4356d..43532575 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::workflows::conformance; use serde_json::json; use tinyflows_storage::DocumentStoreExt; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index 54b6fcf8..3ece6a00 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -64,7 +64,7 @@ async fn one_call_scopes_both_halves() { #[cfg(feature = "storage-drivers")] #[tokio::test] async fn one_document_handle_holds_both_halves_and_survives_reopening() { - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let scoped = MemoryStorage::new() .for_scope(&Scope::local()) .expect("scope"); diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index d63d5911..1377cd20 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -130,7 +130,7 @@ async fn settled_run_updates_its_workflow_summary_only_for_the_winning_status() #[tokio::test] async fn a_run_from_a_removed_flow_incarnation_cannot_stamp_its_replacement() { use crate::catalog::{DEFINITIONS, RUNS, test_support::docs_in}; - use tinyflows_storage::{MemoryStorage, Precondition}; + use tinyflows_storage::{ Precondition}; let storage = MemoryStorage::new(); let store = catalog_in(&storage, "incarnation-summary"); diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index 24bf5b6f..29cd6af1 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -2,15 +2,15 @@ use super::*; use crate::catalog::FlowCatalogDocuments; use serde_json::json; use tinyflows_storage::{ - DocumentStoreExt, MemoryStorage, MigrateOptions, Precondition, Query, Registry, Scope, - ScopedStorage, StorageBackend, WriteOptions, + DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, Scope, + Storage, StorageBackend, WriteOptions, }; -fn scoped() -> ScopedStorage { +fn scoped() -> Storage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() } -async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) { +async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { storage .documents() .put(collection, id, body, Precondition::None) @@ -19,7 +19,7 @@ async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) } async fn seed_with_created_ms( - storage: &ScopedStorage, + storage: &Storage, collection: &str, id: &str, body: Value, @@ -38,7 +38,7 @@ async fn seed_with_created_ms( .unwrap(); } -async fn migrate(storage: &ScopedStorage) { +async fn migrate(storage: &Storage) { let registry = Registry::compose([crate::catalog_storage_schema()]).unwrap(); registry .migrate(storage, &MigrateOptions::new()) diff --git a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs index 9583143c..fcc49604 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs @@ -1,6 +1,6 @@ use super::*; use std::sync::Arc; -use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStore, Precondition}; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs index 0a552306..c4a67090 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs @@ -1,6 +1,6 @@ use super::*; use tinyflows_catalog::FlowRunStep; -use tinyflows_storage::{MemoryStorage, StorageBackend}; +use tinyflows_storage::{ StorageBackend}; async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); diff --git a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs index 2faad879..6989e2c1 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs @@ -1,7 +1,7 @@ use super::*; use serde_json::json; use std::sync::Arc; -use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStore, Precondition}; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; diff --git a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs index 84576d40..86bba113 100644 --- a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs @@ -2,7 +2,7 @@ use super::*; use serde_json::json; use std::sync::Arc; use tinyflows::graph::ids::NodeId; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; #[cfg(feature = "catalog")] use tinyflows_storage::{Precondition, WriteOptions}; diff --git a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs index d1cbf792..cf933540 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs @@ -8,7 +8,7 @@ use crate::checkpoint::DriverCheckpointer; use serde_json::json; use std::sync::Arc; use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; -use tinyflows_storage::{DocumentStore, MemoryStorage, Precondition, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Precondition, Scope, StorageBackend}; fn docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-drivers/src/checkpoint_tests.rs b/crates/tinyflows-drivers/src/checkpoint_tests.rs index f00a0b3c..47fda448 100644 --- a/crates/tinyflows-drivers/src/checkpoint_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_tests.rs @@ -6,9 +6,9 @@ use super::*; use crate::checkpoint_layout::{checkpoint_id_key, subgraph_key, thread_id_key}; use serde_json::json; use tinyflows::graph::ids::NodeId; -use tinyflows_storage::{MemoryStorage, Precondition, Scope, StorageBackend, StorageError}; +use tinyflows_storage::{ Precondition, Scope, StorageBackend, StorageError}; -fn docs(storage: &MemoryStorage, scope: &str) -> Arc { +fn docs(storage: & scope: &str) -> Arc { Arc::clone( storage .for_scope(&Scope::new(scope).unwrap()) diff --git a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs index f0e483a6..81c790fa 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs @@ -4,7 +4,7 @@ use super::test_support::{daily, docs_in, store}; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::Schedule; -use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Query, primitives}; +use tinyflows_storage::{DocumentStoreExt, Query, primitives}; async fn lease_count(store: &CronDocuments) -> usize { store diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index 5dac3a76..f59d9605 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -9,7 +9,7 @@ use tinyflows_schedule::CronJobPatch; use tinyflows_storage::MemoryStorage; fn fault_store( - backend: &MemoryStorage, + backend: & scope: &str, barrier: Option>, ) -> (CronDocuments, Arc) { diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs index 45e0a27c..5c5da078 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs @@ -5,7 +5,7 @@ use serde_json::Value; use std::sync::Arc; use tinyflows_schedule::Schedule; use tinyflows_storage::{ - Capabilities, CollectionSpec, DocumentStore, Filter, MemoryStorage, Page, Precondition, Query, + Capabilities, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Scope, Sort, StorageBackend, Version, Versioned, WriteOp, WriteResult, }; use tokio::sync::Notify; diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index fc9ae7d8..3797eeb2 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -3,15 +3,15 @@ use chrono::TimeZone; use serde_json::json; use tinyflows_schedule::Schedule; use tinyflows_storage::{ - DocumentStoreExt, MemoryStorage, MigrateOptions, Precondition, Query, Registry, Scope, - ScopedStorage, StorageBackend, + DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, Scope, + Storage, StorageBackend, }; -fn scoped() -> ScopedStorage { +fn scoped() -> Storage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() } -async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) { +async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { storage .documents() .put(collection, id, body, Precondition::None) @@ -19,7 +19,7 @@ async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) .unwrap(); } -async fn migrate(storage: &ScopedStorage) { +async fn migrate(storage: &Storage) { Registry::compose([storage_schema()]) .unwrap() .migrate(storage, &MigrateOptions::new()) diff --git a/crates/tinyflows-drivers/src/schedule/mod_tests.rs b/crates/tinyflows-drivers/src/schedule/mod_tests.rs index 2fbc33f1..fe59da98 100644 --- a/crates/tinyflows-drivers/src/schedule/mod_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/mod_tests.rs @@ -3,7 +3,7 @@ use super::*; use super::test_support::{daily, docs_in, store}; use chrono::Utc; use serde_json::json; -use tinyflows_storage::{MemoryStorage, Precondition}; +use tinyflows_storage::{ Precondition}; #[test] fn limits_default_like_the_sqlite_store_and_clamp_to_one() { diff --git a/crates/tinyflows-drivers/src/state_store_tests.rs b/crates/tinyflows-drivers/src/state_store_tests.rs index 5facfbbc..51fa90b6 100644 --- a/crates/tinyflows-drivers/src/state_store_tests.rs +++ b/crates/tinyflows-drivers/src/state_store_tests.rs @@ -1,8 +1,8 @@ use super::*; use serde_json::json; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; -fn docs(storage: &MemoryStorage, scope: &str) -> Arc { +fn docs(storage: & scope: &str) -> Arc { Arc::clone( storage .for_scope(&Scope::new(scope).unwrap()) diff --git a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs index 5907f178..331afa83 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs @@ -1,12 +1,12 @@ use super::*; use serde_json::json; -use tinyflows_storage::{Kv, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{Kv, Scope, StorageBackend}; -fn scoped() -> ScopedStorage { +fn scoped() -> Storage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() } -async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) { +async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { storage .documents() .put(collection, id, body, Precondition::None) diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index 4cd34041..0431a686 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -32,7 +32,7 @@ fn document(id: &str) -> String { .to_string() } -fn store() -> (MemoryStorage, Arc) { +fn store() -> ( Arc) { let storage = MemoryStorage::new(); let store: Arc = Arc::new(store_over(&storage, "local")); (storage, store) diff --git a/crates/tinyflows-drivers/src/store/locks_tests.rs b/crates/tinyflows-drivers/src/store/locks_tests.rs index 44754eb2..a3cec28f 100644 --- a/crates/tinyflows-drivers/src/store/locks_tests.rs +++ b/crates/tinyflows-drivers/src/store/locks_tests.rs @@ -6,7 +6,7 @@ use std::time::Duration; use serde_json::json; use tinyflows::store::WorkflowStore; -use tinyflows_storage::{Key, MemoryStorage, Precondition, primitives}; +use tinyflows_storage::{Key, Precondition, primitives}; use super::DriverWorkflowStore; use super::locks::{LEASE_OWNER, lease_key}; diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index b61b9f7f..2ebe7412 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -3,21 +3,21 @@ use std::sync::Arc; use serde_json::{Value, json}; use tinyflows::store::WorkflowStore; use tinyflows_storage::{ - DocumentStore, DocumentStoreExt, MemoryStorage, MigrateOptions, Precondition, Query, Registry, - Scope, ScopeClass, ScopedStorage, StorageBackend, + DocumentStore, DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, + Scope, ScopeClass, Storage, StorageBackend, }; use super::*; -fn scoped() -> ScopedStorage { +fn scoped() -> Storage { MemoryStorage::new().for_scope(&Scope::local()).unwrap() } -fn docs(storage: &ScopedStorage) -> Arc { +fn docs(storage: &Storage) -> Arc { Arc::clone(storage.documents()) } -async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) { +async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { storage .documents() .put(collection, id, body, Precondition::None) @@ -25,7 +25,7 @@ async fn seed(storage: &ScopedStorage, collection: &str, id: &str, body: Value) .unwrap(); } -async fn migrate(storage: &ScopedStorage) { +async fn migrate(storage: &Storage) { Registry::compose([crate::storage_schema()]) .unwrap() .migrate(storage, &MigrateOptions::new()) diff --git a/crates/tinyflows-drivers/src/store/mod_tests.rs b/crates/tinyflows-drivers/src/store/mod_tests.rs index f05c70c6..1c70e14f 100644 --- a/crates/tinyflows-drivers/src/store/mod_tests.rs +++ b/crates/tinyflows-drivers/src/store/mod_tests.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use serde_json::json; use tinyflows::store::types::{WorkflowDefaults, WorkflowError, WorkflowRecord}; use tinyflows::store::{HostPolicy, WorkflowStore, parse_workflow, require}; -use tinyflows_storage::{MemoryStorage, Precondition}; +use tinyflows_storage::{ Precondition}; use super::test_support::{Faulty, block_on, docs_in, record, store, valid_document}; use super::*; diff --git a/crates/tinyflows-drivers/src/store/notes_tests.rs b/crates/tinyflows-drivers/src/store/notes_tests.rs index aa109f44..f5f80d67 100644 --- a/crates/tinyflows-drivers/src/store/notes_tests.rs +++ b/crates/tinyflows-drivers/src/store/notes_tests.rs @@ -5,7 +5,7 @@ use serde_json::json; use tinyflows::store::types::{NoteKind, NoteSource, WorkflowError, WorkflowNote}; use tinyflows::store::{MAX_NOTES, WorkflowStore, current_notes, mint_note_id}; -use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Precondition, Query}; +use tinyflows_storage::{DocumentStoreExt, Precondition, Query}; use super::test_support::{block_on, docs_in, store, store_over}; use super::{DriverWorkflowStore, NOTES, QUARANTINE}; diff --git a/crates/tinyflows-drivers/src/store/proposals_tests.rs b/crates/tinyflows-drivers/src/store/proposals_tests.rs index 73a5e6dd..03df6cfd 100644 --- a/crates/tinyflows-drivers/src/store/proposals_tests.rs +++ b/crates/tinyflows-drivers/src/store/proposals_tests.rs @@ -9,7 +9,7 @@ use std::sync::atomic::Ordering; use serde_json::json; use tinyflows::store::types::{ProposalStatus, WorkflowError, WorkflowProposal, fingerprint}; use tinyflows::store::{WorkflowStore, mint_proposal_id, require_proposal}; -use tinyflows_storage::{DocumentStoreExt, MemoryStorage, Precondition}; +use tinyflows_storage::{DocumentStoreExt, Precondition}; use super::test_support::{Faulty, block_on, docs_in, record, store, store_over}; use super::{DEFINITIONS, DriverWorkflowStore, PROPOSALS}; diff --git a/crates/tinyflows-drivers/src/store/runs_tests.rs b/crates/tinyflows-drivers/src/store/runs_tests.rs index 2d019a70..4932004f 100644 --- a/crates/tinyflows-drivers/src/store/runs_tests.rs +++ b/crates/tinyflows-drivers/src/store/runs_tests.rs @@ -126,7 +126,7 @@ fn an_executor_and_a_cancel_request_survive_a_write_and_read() { #[test] fn an_unreadable_run_document_is_skipped_by_listings_and_named_by_a_direct_read() { use serde_json::json; - use tinyflows_storage::{MemoryStorage, Precondition}; + use tinyflows_storage::{ Precondition}; let storage = MemoryStorage::new(); let docs = super::test_support::docs_in(&storage, "local"); let store = super::DriverWorkflowStore::new(docs.clone()); diff --git a/crates/tinyflows-module/src/caps_tests/scope_tests.rs b/crates/tinyflows-module/src/caps_tests/scope_tests.rs index c0ecaea6..fe7ef824 100644 --- a/crates/tinyflows-module/src/caps_tests/scope_tests.rs +++ b/crates/tinyflows-module/src/caps_tests/scope_tests.rs @@ -5,7 +5,7 @@ use serde_json::{Value, json}; use std::sync::Arc; use tinyflows_bus::catalog::{self, CatalogStorage}; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; struct DocumentsHost(Arc); diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index 06b64ecd..ebc984c1 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -10,7 +10,7 @@ use tinyflows_bus::catalog::{ PutWorkflowRequest, PutWorkflowResponse, }; use tinyflows_storage::{ - CollectionSpec, DocumentStore, Key, MemoryStorage, Precondition, Scope, StorageBackend, + CollectionSpec, DocumentStore, Key, Precondition, Scope, StorageBackend, WriteOptions, }; diff --git a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs index a8b40b27..4a7ff874 100644 --- a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs +++ b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs @@ -5,7 +5,7 @@ use std::task::Poll; use tinyflows::caps::StateStore as _; use tinyflows_bus::catalog::{CallbackAddress, CatalogStorage}; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; fn storage(path: &str) -> CatalogStorage { CatalogStorage { diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index 15774f34..65a18205 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -12,7 +12,7 @@ use tinyflows_bus::runs::{ }; use tinyflows_catalog::FlowRunStep; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; use tokio::sync::oneshot; #[tokio::test] diff --git a/crates/tinyflows-module/src/run_manager_tests.rs b/crates/tinyflows-module/src/run_manager_tests.rs index f4ee038c..d9d179d9 100644 --- a/crates/tinyflows-module/src/run_manager_tests.rs +++ b/crates/tinyflows-module/src/run_manager_tests.rs @@ -1,7 +1,7 @@ use super::*; use tinyflows_bus::catalog::CallbackAddress; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; fn storage(owner: &str) -> tinyflows_bus::catalog::CatalogStorage { tinyflows_bus::catalog::CatalogStorage { diff --git a/crates/tinyflows-module/src/schedule_clock_tests.rs b/crates/tinyflows-module/src/schedule_clock_tests.rs index 9cc196ca..4249233b 100644 --- a/crates/tinyflows-module/src/schedule_clock_tests.rs +++ b/crates/tinyflows-module/src/schedule_clock_tests.rs @@ -17,7 +17,7 @@ async fn retry_sleep_uses_tokio_time_without_changing_the_calendar_instant() { #[tokio::test] async fn a_manual_pass_uses_the_supplied_time_for_due_jobs_and_run_records() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host @@ -161,7 +161,7 @@ fn a_host_clock_reads_system_time_until_the_host_sets_an_instant() { #[tokio::test] async fn ticking_at_a_later_instant_runs_a_future_job_without_waiting_for_it() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index 296dea15..7ef158ef 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -2,7 +2,7 @@ use super::*; use std::sync::Arc; use tinyflows_storage::{ - Capability, ErrorKind, MemoryStorage, Scope, StorageBackend, WriteOp, + Capability, ErrorKind, Scope, StorageBackend, WriteOp, }; struct Callback(Arc); diff --git a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs index 278a827c..3feeae2e 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs @@ -6,7 +6,7 @@ async fn prepared_manual_worker_stop_wait_and_ack_are_idempotent_and_fenced() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, StoreConfig, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -278,7 +278,7 @@ async fn delivery_projection_error_releases_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; state diff --git a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs index facc9712..1d9bd7a4 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs @@ -18,7 +18,7 @@ async fn claim_reply_lost_after_commit_is_reconciled_and_released_before_termina ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -134,7 +134,7 @@ async fn release_reply_loss_reports_cleanup_pending_until_retry_confirms_fence() ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -273,7 +273,7 @@ async fn unsupported_claim_backend_fails_before_recording_claim_intent() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -387,7 +387,7 @@ async fn terminal_commit_error_releases_queued_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index 97869e4d..e7255d43 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -125,7 +125,7 @@ async fn separate_schedule_object_parses_delays_and_rejects_cross_tenant_deliver #[tokio::test] async fn run_job_now_is_a_typed_module_owned_admission_operation() { use tinyflows_bus::schedule::{ManualDelivery, RunJobNow, RunJobReply, StoreConfig}; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, state) = connections().await; let backend = MemoryStorage::new(); @@ -222,7 +222,7 @@ async fn run_job_now_returns_typed_refusal_when_a_scheduled_claim_is_active() { use tinyflows_bus::schedule::{ ManualDelivery, RunJobNow, RunJobRefusal, RunJobReply, StoreConfig, }; - use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{ Scope, StorageBackend}; let (_server, host, _) = connections().await; let backend = MemoryStorage::new(); @@ -425,7 +425,7 @@ mod queue_tests; #[tokio::test] async fn module_document_store_retries_policy_migration_conflicts_without_losing_seeded_jobs() { - use tinyflows_storage::{MemoryStorage, Precondition, Scope, StorageBackend}; + use tinyflows_storage::{ Precondition, Scope, StorageBackend}; let (_server, host, _) = connections().await; let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); let docs = scoped.documents().clone(); diff --git a/crates/tinyflows-module/src/schedule_store_tests.rs b/crates/tinyflows-module/src/schedule_store_tests.rs index d20570c9..1ddf694a 100644 --- a/crates/tinyflows-module/src/schedule_store_tests.rs +++ b/crates/tinyflows-module/src/schedule_store_tests.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use std::time::Duration; use tinyflows_schedule::runtime::*; use tinyflows_schedule::{DeliveryStatus, Schedule}; -use tinyflows_storage::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{ Scope, StorageBackend}; use tokio::sync::Notify; struct Replacement(CronDocuments); diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 89beb90d..201d86cf 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -3,7 +3,7 @@ //! These helpers contain record and migration behavior owned by TinyFlows; //! backend selection stays with the host. -pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, async_trait}; +pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, Storage, StorageBackend, async_trait}; pub use tinystoragedrivers_types::*; mod key; mod schema; From 21d04dc25173c67442c7829afe8808f5c7c3d18b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:22:35 +0300 Subject: [PATCH 16/64] chore: files changed crates/tinyflows-adaptive/src/drivers/ledger_tests.rs,crates/tinyflows-adaptive Auto-committed-on: dragonfly --- crates/tinyflows-adaptive/src/drivers/ledger_tests.rs | 1 + crates/tinyflows-adaptive/src/drivers/migrate_tests.rs | 1 + crates/tinyflows-adaptive/src/drivers/vault_tests.rs | 1 + crates/tinyflows-adaptive/src/storage_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/definitions_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/drafts_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/kv_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/migrate_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs | 1 + .../tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs | 1 + crates/tinyflows-drivers/src/catalog/test_support.rs | 1 + crates/tinyflows-drivers/src/checkpoint_import_tests.rs | 1 + crates/tinyflows-drivers/src/checkpoint_layout_tests.rs | 1 + crates/tinyflows-drivers/src/checkpoint_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/admission_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/claims_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/import_tests.rs | 1 + .../tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/migrate_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/mod_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs | 1 + crates/tinyflows-drivers/src/schedule/test_support.rs | 1 + crates/tinyflows-drivers/src/state_store_tests.rs | 1 + crates/tinyflows-drivers/src/storage/kv_import_tests.rs | 1 + crates/tinyflows-drivers/src/store/authoring_tests.rs | 1 + crates/tinyflows-drivers/src/store/concurrency_tests.rs | 1 + crates/tinyflows-drivers/src/store/history_tests.rs | 1 + crates/tinyflows-drivers/src/store/import_tests.rs | 1 + crates/tinyflows-drivers/src/store/locks_tests.rs | 1 + crates/tinyflows-drivers/src/store/migrate_tests.rs | 1 + crates/tinyflows-drivers/src/store/mod_tests.rs | 1 + crates/tinyflows-drivers/src/store/notes_tests.rs | 1 + crates/tinyflows-drivers/src/store/proposals_tests.rs | 1 + crates/tinyflows-drivers/src/store/runs_tests.rs | 1 + crates/tinyflows-drivers/src/store/test_support.rs | 1 + crates/tinyflows-module/src/caps_tests/scope_tests.rs | 1 + crates/tinyflows-module/src/catalog_service_tests.rs | 1 + .../tinyflows-module/src/run_manager/dedup_settlement_tests.rs | 1 + .../src/run_manager/events_persistence_test_support.rs | 1 + crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs | 1 + crates/tinyflows-module/src/run_manager_tests.rs | 1 + crates/tinyflows-module/src/schedule_clock_tests.rs | 1 + crates/tinyflows-module/src/schedule_documents_tests.rs | 1 + .../src/schedule_manual_worker_lifecycle_service_tests.rs | 1 + .../tinyflows-module/src/schedule_manual_worker_service_tests.rs | 1 + crates/tinyflows-module/src/schedule_members_tests.rs | 1 + crates/tinyflows-module/src/schedule_service_tests.rs | 1 + crates/tinyflows-module/src/schedule_store_tests.rs | 1 + 54 files changed, 54 insertions(+) diff --git a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs index d554e206..1b3f1542 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::drivers::key; use crate::execute::StepRecord; use crate::ledger::{Ledger, Score, conformance}; diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index d6331cbf..698736a2 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use serde_json::{Value, json}; diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index 43532575..083e264a 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::workflows::conformance; use serde_json::json; use tinyflows_storage::DocumentStoreExt; diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index 3ece6a00..a3c756c5 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[test] fn memory_has_to_be_asked_for_by_name() { diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index 1377cd20..fa6c5a2f 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in, trigger_graph}; use std::sync::Arc; use tinyflows_storage::MemoryStorage; diff --git a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs index 14b54bf7..0d2f29cc 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in}; use tinyflows_storage::MemoryStorage; diff --git a/crates/tinyflows-drivers/src/catalog/kv_tests.rs b/crates/tinyflows-drivers/src/catalog/kv_tests.rs index 112d4510..599ea310 100644 --- a/crates/tinyflows-drivers/src/catalog/kv_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/kv_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in}; use serde_json::json; use tinyflows_storage::MemoryStorage; diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index 29cd6af1..66d25d78 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::FlowCatalogDocuments; use serde_json::json; use tinyflows_storage::{ diff --git a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs index fcc49604..abc8fb54 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows_storage::{DocumentStore, Precondition}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs index bc1b5765..9aeeb756 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A run lifecycle driven through two catalog handles on one backend. use super::*; diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs index c4a67090..194429aa 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows_catalog::FlowRunStep; use tinyflows_storage::{ StorageBackend}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs index 6989e2c1..1faff159 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use std::sync::Arc; use tinyflows_storage::{DocumentStore, Precondition}; diff --git a/crates/tinyflows-drivers/src/catalog/test_support.rs b/crates/tinyflows-drivers/src/catalog/test_support.rs index 31494908..72c38c72 100644 --- a/crates/tinyflows-drivers/src/catalog/test_support.rs +++ b/crates/tinyflows-drivers/src/catalog/test_support.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures shared by the catalog tests. use std::sync::Arc; diff --git a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs index 86bba113..22a4d252 100644 --- a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use std::sync::Arc; use tinyflows::graph::ids::NodeId; diff --git a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs index cf933540..82cbbdc9 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Pins the checkpointer's documents to the layout `tinyagents-graph` stores //! for namespace `flows`. The fixtures are copied from tinyagents' own tests //! (`checkpoint/drivers_tests.rs`, `storage/mod_tests.rs` on its epic branch) diff --git a/crates/tinyflows-drivers/src/checkpoint_tests.rs b/crates/tinyflows-drivers/src/checkpoint_tests.rs index 47fda448..a2fec36c 100644 --- a/crates/tinyflows-drivers/src/checkpoint_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Behaviour of the storage-driver checkpointer, run against a driver document //! store: the checkpointer contract every durable backend is held to, plus the //! isolation and key-encoding guarantees the driver layout adds. diff --git a/crates/tinyflows-drivers/src/schedule/admission_tests.rs b/crates/tinyflows-drivers/src/schedule/admission_tests.rs index dbab8fb1..a7b20064 100644 --- a/crates/tinyflows-drivers/src/schedule/admission_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/admission_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Shared admission policy fences both configured and existing handles. use super::{ CronDocuments, diff --git a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs index 56eaae2a..ad7c7ac7 100644 --- a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A completed receipt removes only its own unchanged incarnation. use super::{ CronDocuments, diff --git a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs index 81c790fa..64a0f2e4 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Occurrence leases are compacted once the job has moved past them, without //! weakening the fencing of stale workers. use super::test_support::{daily, docs_in, store}; diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index f59d9605..46801172 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A claim's lease and the job document are two writes; these pin down that a //! holder which lost its claim cannot commit through the gap between them, and //! that no error path strands a lease. diff --git a/crates/tinyflows-drivers/src/schedule/claims_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_tests.rs index 4171e04e..89b9a38b 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Occurrences are leased across handles and fenced against stale workers. use super::{ CronDocuments, diff --git a/crates/tinyflows-drivers/src/schedule/import_tests.rs b/crates/tinyflows-drivers/src/schedule/import_tests.rs index edf205b5..5a6fb298 100644 --- a/crates/tinyflows-drivers/src/schedule/import_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/import_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::schedule::job_not_found; use crate::schedule::test_support::{channel_origin, daily, docs_in, store}; diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs index 5c5da078..b08b5930 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use async_trait::async_trait; use chrono::{Duration, Utc}; use serde_json::Value; diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs index 6ae138ee..ea28a698 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::docs_in; use super::*; use chrono::{Duration, Utc}; diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index 3797eeb2..35f08583 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use chrono::TimeZone; use serde_json::json; use tinyflows_schedule::Schedule; diff --git a/crates/tinyflows-drivers/src/schedule/mod_tests.rs b/crates/tinyflows-drivers/src/schedule/mod_tests.rs index fe59da98..60e203ff 100644 --- a/crates/tinyflows-drivers/src/schedule/mod_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/mod_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{daily, docs_in, store}; use chrono::Utc; diff --git a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs index b5e86c9d..fdf0e86a 100644 --- a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Removal compatibility and admission bookkeeping across supported backends. use super::super::{ CronDocuments, JOBS, diff --git a/crates/tinyflows-drivers/src/schedule/test_support.rs b/crates/tinyflows-drivers/src/schedule/test_support.rs index 32778aa1..9d717215 100644 --- a/crates/tinyflows-drivers/src/schedule/test_support.rs +++ b/crates/tinyflows-drivers/src/schedule/test_support.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures the schedule tests share. use std::sync::Arc; diff --git a/crates/tinyflows-drivers/src/state_store_tests.rs b/crates/tinyflows-drivers/src/state_store_tests.rs index 51fa90b6..61e5afbb 100644 --- a/crates/tinyflows-drivers/src/state_store_tests.rs +++ b/crates/tinyflows-drivers/src/state_store_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows_storage::{ Scope, StorageBackend}; diff --git a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs index 331afa83..cd67a128 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows_storage::{Kv, Scope, StorageBackend}; diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index 0431a686..5609c68f 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Tests for patch-based workflow editing. //! //! The theme is that a bad edit costs an error message, never the saved diff --git a/crates/tinyflows-drivers/src/store/concurrency_tests.rs b/crates/tinyflows-drivers/src/store/concurrency_tests.rs index ece62d80..519d8d9a 100644 --- a/crates/tinyflows-drivers/src/store/concurrency_tests.rs +++ b/crates/tinyflows-drivers/src/store/concurrency_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Racing writers: the compare-and-swap on a definition and the revision //! captured beside it must hold up when two threads, or two store instances, //! write at once. diff --git a/crates/tinyflows-drivers/src/store/history_tests.rs b/crates/tinyflows-drivers/src/store/history_tests.rs index f9cd83c7..747b8e8e 100644 --- a/crates/tinyflows-drivers/src/store/history_tests.rs +++ b/crates/tinyflows-drivers/src/store/history_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Revision snapshotting, undo, and rollback: saving over a workflow snapshots //! what it replaced, undo restores (and is itself undoable), a named rollback //! restores exactly that revision, and history never leaks into the ordinary diff --git a/crates/tinyflows-drivers/src/store/import_tests.rs b/crates/tinyflows-drivers/src/store/import_tests.rs index a04e8ff1..cab8fab8 100644 --- a/crates/tinyflows-drivers/src/store/import_tests.rs +++ b/crates/tinyflows-drivers/src/store/import_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{ NoteKind, NoteSource, ProposalStatus, RunStatus, WorkflowError, WorkflowNote, WorkflowProposal, diff --git a/crates/tinyflows-drivers/src/store/locks_tests.rs b/crates/tinyflows-drivers/src/store/locks_tests.rs index a3cec28f..eb5a2279 100644 --- a/crates/tinyflows-drivers/src/store/locks_tests.rs +++ b/crates/tinyflows-drivers/src/store/locks_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! The decision lease: exclusion across store instances, release on drop, and //! recovery from a holder that went away. diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index 2ebe7412..a1c3089d 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use serde_json::{Value, json}; diff --git a/crates/tinyflows-drivers/src/store/mod_tests.rs b/crates/tinyflows-drivers/src/store/mod_tests.rs index 1c70e14f..bca89c03 100644 --- a/crates/tinyflows-drivers/src/store/mod_tests.rs +++ b/crates/tinyflows-drivers/src/store/mod_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Save/delete round-tripping: the graph and host fields survive a save, the //! `defaults` block round-trips, an uncompilable graph is refused, ids are //! guarded, and the host policy judges reads as well as writes. diff --git a/crates/tinyflows-drivers/src/store/notes_tests.rs b/crates/tinyflows-drivers/src/store/notes_tests.rs index f5f80d67..58881805 100644 --- a/crates/tinyflows-drivers/src/store/notes_tests.rs +++ b/crates/tinyflows-drivers/src/store/notes_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! The journal on documents: ordering, scoping, supersession, the cap, and what //! happens when things go wrong: a journal that cannot be parsed, an id that is //! not usable, concurrent appenders. diff --git a/crates/tinyflows-drivers/src/store/proposals_tests.rs b/crates/tinyflows-drivers/src/store/proposals_tests.rs index 03df6cfd..d27815fb 100644 --- a/crates/tinyflows-drivers/src/store/proposals_tests.rs +++ b/crates/tinyflows-drivers/src/store/proposals_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Proposals on documents. A proposal is the one artifact in this feature that //! can change a saved graph, so what is asserted here is mostly about *not* //! doing that by accident: a listing that cannot leak another workflow's diff --git a/crates/tinyflows-drivers/src/store/runs_tests.rs b/crates/tinyflows-drivers/src/store/runs_tests.rs index 4932004f..5b4dc002 100644 --- a/crates/tinyflows-drivers/src/store/runs_tests.rs +++ b/crates/tinyflows-drivers/src/store/runs_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Run-record listing: newest first, scoped to their workflow, and a run that //! was never recorded is a distinct error rather than a silent `None`. diff --git a/crates/tinyflows-drivers/src/store/test_support.rs b/crates/tinyflows-drivers/src/store/test_support.rs index a129b318..38ab77b9 100644 --- a/crates/tinyflows-drivers/src/store/test_support.rs +++ b/crates/tinyflows-drivers/src/store/test_support.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures the store tests share. use std::sync::Arc; diff --git a/crates/tinyflows-module/src/caps_tests/scope_tests.rs b/crates/tinyflows-module/src/caps_tests/scope_tests.rs index fe7ef824..55e80d15 100644 --- a/crates/tinyflows-module/src/caps_tests/scope_tests.rs +++ b/crates/tinyflows-module/src/caps_tests/scope_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index ebc984c1..09ca197a 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; use std::sync::{ diff --git a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs index 4a7ff874..3ffd106e 100644 --- a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs +++ b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::future::Future as _; use std::task::Poll; diff --git a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs index 9230cc9e..d344a3fa 100644 --- a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs +++ b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use async_trait::async_trait; use serde_json::{Value, json}; use std::sync::{ diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index 65a18205..511f8d86 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; diff --git a/crates/tinyflows-module/src/run_manager_tests.rs b/crates/tinyflows-module/src/run_manager_tests.rs index d9d179d9..91521e23 100644 --- a/crates/tinyflows-module/src/run_manager_tests.rs +++ b/crates/tinyflows-module/src/run_manager_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows_bus::catalog::CallbackAddress; use tinyflows_drivers::catalog::FlowCatalogDocuments; use tinyflows_storage::{ Scope, StorageBackend}; diff --git a/crates/tinyflows-module/src/schedule_clock_tests.rs b/crates/tinyflows-module/src/schedule_clock_tests.rs index 4249233b..72fcd1ef 100644 --- a/crates/tinyflows-module/src/schedule_clock_tests.rs +++ b/crates/tinyflows-module/src/schedule_clock_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; // Manual passes use one supplied calendar instant across dispatch and records. use chrono::{Duration, Utc}; diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index 7ef158ef..be80270d 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows_storage::{ diff --git a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs index 3feeae2e..9048e68b 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn prepared_manual_worker_stop_wait_and_ack_are_idempotent_and_fenced() { diff --git a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs index 1d9bd7a4..de53874b 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[cfg(test)] #[path = "schedule_manual_worker_stop_identity_tests.rs"] diff --git a/crates/tinyflows-module/src/schedule_members_tests.rs b/crates/tinyflows-module/src/schedule_members_tests.rs index be3f6232..febd80e8 100644 --- a/crates/tinyflows-module/src/schedule_members_tests.rs +++ b/crates/tinyflows-module/src/schedule_members_tests.rs @@ -1,3 +1,4 @@ +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Every contract-v3 pure member, called through the bus like a host would. use super::tests::{connections, document_callbacks}; use super::*; diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index e7255d43..52c576c9 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows_bus::schedule::{CALLBACK_INTERFACE, CallbackAddress, CronJob}; use tokio::sync::Mutex; diff --git a/crates/tinyflows-module/src/schedule_store_tests.rs b/crates/tinyflows-module/src/schedule_store_tests.rs index 1ddf694a..805a085a 100644 --- a/crates/tinyflows-module/src/schedule_store_tests.rs +++ b/crates/tinyflows-module/src/schedule_store_tests.rs @@ -1,4 +1,5 @@ use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use async_trait::async_trait; use chrono::{DateTime, Utc}; use std::time::Duration; From 66d408ba3889342f673bbf2fc6505da12bde1530 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:22:59 +0300 Subject: [PATCH 17/64] refactor(tests): update storage backend usage in tests Replaces direct usage of `tinyflows_storage::MemoryStorage` with `tinystoragedrivers_testkit::MemoryBackend` across various test files. This aligns tests with the current storage driver implementation. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/catalog/definitions_tests.rs | 1 - crates/tinyflows-drivers/src/catalog/drafts_tests.rs | 1 - crates/tinyflows-drivers/src/catalog/kv_tests.rs | 1 - crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs | 3 +-- crates/tinyflows-drivers/src/schedule/admission_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/claims_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/import_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs | 1 - crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs | 1 - crates/tinyflows-drivers/src/store/authoring_tests.rs | 1 - crates/tinyflows-drivers/src/store/concurrency_tests.rs | 1 - crates/tinyflows-drivers/src/store/history_tests.rs | 1 - crates/tinyflows-drivers/src/store/import_tests.rs | 1 - crates/tinyflows-module/src/schedule_members_tests.rs | 2 +- crates/tinyflows-module/src/schedule_service_tests.rs | 2 +- 17 files changed, 3 insertions(+), 18 deletions(-) diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index fa6c5a2f..ad0761ff 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -2,7 +2,6 @@ use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in, trigger_graph}; use std::sync::Arc; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn create_get_list_delete_roundtrip() { diff --git a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs index 0d2f29cc..df0bd47e 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs @@ -1,7 +1,6 @@ use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in}; -use tinyflows_storage::MemoryStorage; fn graph() -> Value { json!({ "nodes": [{ "id": "t", "kind": "trigger" }], "edges": [] }) diff --git a/crates/tinyflows-drivers/src/catalog/kv_tests.rs b/crates/tinyflows-drivers/src/catalog/kv_tests.rs index 599ea310..bb2381cc 100644 --- a/crates/tinyflows-drivers/src/catalog/kv_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/kv_tests.rs @@ -2,7 +2,6 @@ use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{catalog, catalog_in}; use serde_json::json; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn values_round_trip_per_namespace() { diff --git a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs index 9aeeb756..f4ddfa36 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs @@ -20,8 +20,7 @@ fn step(node: &str) -> FlowRunStep { async fn two_handles_on_one_backend_agree_on_a_run_lifecycle() { use crate::catalog::FlowStateDocuments; use tinyflows::caps::StateStore; - use tinyflows_storage::MemoryStorage; - + let storage = MemoryStorage::new(); let a = crate::catalog::test_support::catalog_in(&storage, "tenant"); let b = crate::catalog::test_support::catalog_in(&storage, "tenant"); diff --git a/crates/tinyflows-drivers/src/schedule/admission_tests.rs b/crates/tinyflows-drivers/src/schedule/admission_tests.rs index a7b20064..7fe87a39 100644 --- a/crates/tinyflows-drivers/src/schedule/admission_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/admission_tests.rs @@ -5,7 +5,6 @@ use super::{ test_support::{daily, docs_in}, }; use tinyflows_schedule::CronJobPatch; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn an_existing_unlimited_handle_cannot_bypass_installed_policy() { diff --git a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs index ad7c7ac7..d626ab16 100644 --- a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs @@ -5,7 +5,6 @@ use super::{ test_support::{daily, docs_in}, }; use chrono::Duration; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn completed_receipt_cannot_remove_a_recreated_job() { diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index 46801172..10bf6be0 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -7,7 +7,6 @@ use super::test_support::{FaultDocuments, daily, docs_in}; use chrono::Duration; use std::sync::{Arc, atomic::AtomicBool}; use tinyflows_schedule::CronJobPatch; -use tinyflows_storage::MemoryStorage; fn fault_store( backend: & diff --git a/crates/tinyflows-drivers/src/schedule/claims_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_tests.rs index 89b9a38b..b1591da6 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_tests.rs @@ -5,7 +5,6 @@ use super::{ test_support::{daily, docs_in}, }; use chrono::Duration; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn two_handles_cannot_claim_the_same_occurrence() { diff --git a/crates/tinyflows-drivers/src/schedule/import_tests.rs b/crates/tinyflows-drivers/src/schedule/import_tests.rs index 5a6fb298..ecbefd25 100644 --- a/crates/tinyflows-drivers/src/schedule/import_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/import_tests.rs @@ -5,7 +5,6 @@ use crate::schedule::job_not_found; use crate::schedule::test_support::{channel_origin, daily, docs_in, store}; use chrono::{TimeZone, Utc}; use tinyflows_schedule::{DeliveryStatus, Schedule}; -use tinyflows_storage::MemoryStorage; /// The JSON form, for comparing jobs (`CronJob` is not `PartialEq`). fn j(value: &T) -> serde_json::Value { diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs index ea28a698..7917af49 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs @@ -3,7 +3,6 @@ use super::test_support::docs_in; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::{CronJobPatch, DeliveryStatus, Schedule}; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn reconciles_manual_claim_receipt_after_an_ambiguous_claim_reply() { diff --git a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs index fdf0e86a..6401916a 100644 --- a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs @@ -5,7 +5,6 @@ use super::super::{ test_support::{FaultDocuments, daily, docs_in}, }; use std::sync::{Arc, atomic::AtomicBool}; -use tinyflows_storage::MemoryStorage; #[tokio::test] async fn nontransactional_deduplication_and_clear_keep_legacy_behavior() { diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index 5609c68f..81a23f7a 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -14,7 +14,6 @@ use tinyflows::store::{ GraphHandle, WorkflowStore, apply_workflow_ops, apply_workflow_ops_observed, create_workflow, preview_workflow_ops, validate_handle, }; -use tinyflows_storage::MemoryStorage; use super::test_support::store_over; diff --git a/crates/tinyflows-drivers/src/store/concurrency_tests.rs b/crates/tinyflows-drivers/src/store/concurrency_tests.rs index 519d8d9a..b4144f86 100644 --- a/crates/tinyflows-drivers/src/store/concurrency_tests.rs +++ b/crates/tinyflows-drivers/src/store/concurrency_tests.rs @@ -6,7 +6,6 @@ use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::{Arc, Barrier}; use tinyflows::store::WorkflowStore; -use tinyflows_storage::MemoryStorage; use super::test_support::{record, store, store_over}; diff --git a/crates/tinyflows-drivers/src/store/history_tests.rs b/crates/tinyflows-drivers/src/store/history_tests.rs index 747b8e8e..d2a81f7b 100644 --- a/crates/tinyflows-drivers/src/store/history_tests.rs +++ b/crates/tinyflows-drivers/src/store/history_tests.rs @@ -7,7 +7,6 @@ use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{record, store, store_over}; use tinyflows::store::types::WorkflowError; use tinyflows::store::{MAX_REVISIONS, WorkflowStore, require, rollback, undo_last}; -use tinyflows_storage::MemoryStorage; #[test] fn a_workflow_saved_for_the_first_time_has_no_history_to_go_back_to() { diff --git a/crates/tinyflows-drivers/src/store/import_tests.rs b/crates/tinyflows-drivers/src/store/import_tests.rs index cab8fab8..73f28561 100644 --- a/crates/tinyflows-drivers/src/store/import_tests.rs +++ b/crates/tinyflows-drivers/src/store/import_tests.rs @@ -5,7 +5,6 @@ use tinyflows::store::types::{ WorkflowRevision, }; use tinyflows::store::{MAX_NOTES, MAX_REVISIONS, WorkflowStore, mint_note_id, new_run_record}; -use tinyflows_storage::MemoryStorage; use crate::store::test_support::{record, store, store_over}; diff --git a/crates/tinyflows-module/src/schedule_members_tests.rs b/crates/tinyflows-module/src/schedule_members_tests.rs index febd80e8..740f6b54 100644 --- a/crates/tinyflows-module/src/schedule_members_tests.rs +++ b/crates/tinyflows-module/src/schedule_members_tests.rs @@ -205,7 +205,7 @@ async fn store_admission_cap_and_import_operations_are_served() { document_callbacks::Documents { docs: { use tinyflows_storage::StorageBackend; - tinyflows_storage::MemoryStorage::new() + tinystoragedrivers_testkit::MemoryBackend::new() .for_scope(&tinyflows_storage::Scope::local()) .unwrap() .documents() diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index 52c576c9..631ed697 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -382,7 +382,7 @@ async fn module_job_storage_preserves_policy_updates_and_run_history() { document_callbacks::Documents { docs: { use tinyflows_storage::StorageBackend; - tinyflows_storage::MemoryStorage::new() + tinystoragedrivers_testkit::MemoryBackend::new() .for_scope(&tinyflows_storage::Scope::local()) .unwrap() .documents() From 8492e2c2e9ce6b4ae0544d09c90d3fb8f707c319 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:24:35 +0300 Subject: [PATCH 18/64] chore: files changed crates/tinyflows-storage/src/schema.rs Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index a83a5244..030eb584 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; use tinystoragedrivers_types::{ - CollectionSpec, Filter, Meta, Precondition, Query, Result, StorageError, Versioned, + CollectionSpec, Filter, Meta, Precondition, Query, Result, StorageError, WriteOptions, SCHEMA_FIELD, }; @@ -115,7 +115,7 @@ impl Step { } /// One owner’s collections and migrations, opened by that owner. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct Schema { /// Storage owner for these collections. pub owner: Owner, @@ -171,7 +171,7 @@ impl Schema { if migrations.done(self.owner.as_str(), step.id).await? { report.already_done += 1; continue; } let progress_id = Key::of([self.owner.as_str(), step.id])?.as_str().to_owned(); let progress = migrations.docs.get_as::(&migrations.collection, &progress_id).await?; - let mut query = Query::filter(Filter::all()).limit(options.batch_size); + let mut query = Query::filter(Filter::default()).limit(options.batch_size); if let Some(cursor) = progress.and_then(|p| p.doc.cursor) { query = query.after(tinystoragedrivers_types::Cursor(cursor)); } From f0937cdfd4dc7c03a6a940f81d1f2b71e3bed9e3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:26:11 +0300 Subject: [PATCH 19/64] refactor(storage): improve migration query and update logic This change updates the migration process to include sorting when querying documents and refines the logic for updating and deleting records. It now filters documents based on schema version during input collection and uses version preconditions for more robust updates and deletions. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema.rs | 28 +++++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index 030eb584..a134a7f1 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; use tinystoragedrivers_types::{ - CollectionSpec, Filter, Meta, Precondition, Query, Result, StorageError, + CollectionSpec, ErrorKind, Filter, Meta, Precondition, Query, Result, Sort, StorageError, WriteOptions, SCHEMA_FIELD, }; @@ -171,7 +171,9 @@ impl Schema { if migrations.done(self.owner.as_str(), step.id).await? { report.already_done += 1; continue; } let progress_id = Key::of([self.owner.as_str(), step.id])?.as_str().to_owned(); let progress = migrations.docs.get_as::(&migrations.collection, &progress_id).await?; - let mut query = Query::filter(Filter::default()).limit(options.batch_size); + let mut query = Query::filter(Filter::default()) + .sort(Sort::asc("_id")) + .limit(options.batch_size); if let Some(cursor) = progress.and_then(|p| p.doc.cursor) { query = query.after(tinystoragedrivers_types::Cursor(cursor)); } @@ -181,16 +183,23 @@ impl Schema { let page = docs.query(source, &query).await?; if page.items.is_empty() { break; } let last_cursor = page.next.clone(); - let inputs = page.items.iter().map(|row| SourceRecord { id: row.id.clone(), body: row.doc.clone(), meta: row.meta.clone() }).collect(); + let inputs = page.items.iter() + .filter(|row| row.meta.schema < step.to) + .map(|row| SourceRecord { id: row.id.clone(), body: row.doc.clone(), meta: row.meta.clone() }) + .collect(); let outputs = (step.run)(inputs)?; let output_ids: std::collections::HashSet<(String, String)> = outputs.iter().map(|r| (r.collection.clone(), r.id.clone())).collect(); for mut target in outputs { if !target.body.is_object() { return Err(StorageError::invalid_input("migration output must be an object")); } target.body[SCHEMA_FIELD] = step.to.into(); let options = target.created_ms.map_or_else(WriteOptions::new, |ms| WriteOptions::new().created_ms(ms)); - match docs.put_with(&target.collection, &target.id, target.body.clone(), Precondition::Absent, options).await { + let source_row = (source == step.item) + .then(|| page.items.iter().find(|row| row.id == target.id)); + let precondition = source_row.flatten() + .map_or(Precondition::Absent, |row| Precondition::Version(row.version)); + match docs.put_with(&target.collection, &target.id, target.body.clone(), precondition, options).await { Ok(_) => written += 1, - Err(error) if error.kind() == tinystoragedrivers_types::ErrorKind::Conflict => { + Err(error) if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => { let existing = docs.get(&target.collection, &target.id).await?; if existing.is_none_or(|row| row.doc != target.body) { return Err(error); } written += 1; @@ -198,11 +207,10 @@ impl Schema { Err(error) => return Err(error), } } - if source != step.item { - for row in &page.items { - if output_ids.contains(&(step.item.to_owned(), row.id.clone())) { - docs.delete(source, &row.id, Precondition::Version(row.version)).await?; - } + for row in &page.items { + if row.meta.schema >= step.to { continue; } + if source != step.item || !output_ids.contains(&(step.item.to_owned(), row.id.clone())) { + docs.delete(source, &row.id, Precondition::Version(row.version)).await?; } } let cursor = last_cursor.map(|cursor| cursor.0); From 1299726abd0534a0e990c72d0d552287fa774880 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:49:59 +0300 Subject: [PATCH 20/64] chore: files changed .github/workflows/ci.yml,Cargo.lock,Cargo.toml,crates/tinyflows-adaptive/Cargo. Auto-committed-on: dragonfly --- .github/workflows/ci.yml | 47 +-- Cargo.lock | 197 +++++---- Cargo.toml | 2 +- crates/tinyflows-adaptive/Cargo.toml | 16 +- .../src/drivers/ledger_tests.rs | 4 +- .../tinyflows-adaptive/src/drivers/migrate.rs | 31 +- .../src/drivers/migrate_tests.rs | 44 +- crates/tinyflows-adaptive/src/drivers/mod.rs | 6 +- .../src/drivers/vault_tests.rs | 4 +- .../tinyflows-adaptive/src/storage_tests.rs | 4 +- .../tests/driver/driver_part_01_tests.rs | 2 +- .../tests/support/driver_store.rs | 2 +- crates/tinyflows-drivers/Cargo.toml | 6 +- .../src/catalog/definitions_tests.rs | 2 +- .../src/catalog/drafts_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/kv_tests.rs | 2 +- .../src/catalog/migrate_tests.rs | 21 +- .../src/catalog/revisions_gap_tests.rs | 2 +- .../src/catalog/runs_lifecycle_tests.rs | 3 +- .../src/catalog/runs_recovery_lease_tests.rs | 6 +- .../src/catalog/runs_replace_tests.rs | 2 +- .../src/catalog/test_support.rs | 5 +- .../src/checkpoint_import_tests.rs | 2 +- .../src/checkpoint_layout_tests.rs | 5 +- .../tinyflows-drivers/src/checkpoint_tests.rs | 5 +- .../src/schedule/admission_tests.rs | 3 +- .../src/schedule/claim_removal_tests.rs | 3 +- .../src/schedule/claims_compaction_tests.rs | 3 +- .../src/schedule/claims_safety_tests.rs | 5 +- .../src/schedule/claims_tests.rs | 3 +- .../src/schedule/import_tests.rs | 2 +- .../src/schedule/manual_claim_fence_tests.rs | 6 +- .../tinyflows-drivers/src/schedule/migrate.rs | 6 +- .../src/schedule/migrate_tests.rs | 19 +- .../src/schedule/mod_tests.rs | 2 +- .../src/schedule/removal_claims_tests.rs | 3 +- .../src/schedule/test_support.rs | 5 +- .../src/state_store_tests.rs | 4 +- .../src/storage/kv_import.rs | 3 +- .../src/storage/kv_import_tests.rs | 14 +- crates/tinyflows-drivers/src/storage/mod.rs | 13 +- .../src/store/authoring_tests.rs | 5 +- .../src/store/concurrency_tests.rs | 3 +- .../src/store/history_tests.rs | 3 +- .../src/store/locks_tests.rs | 3 +- .../src/store/migrate_tests.rs | 17 +- .../tinyflows-drivers/src/store/mod_tests.rs | 3 +- .../src/store/notes_tests.rs | 3 +- .../src/store/proposals_tests.rs | 3 +- .../tinyflows-drivers/src/store/runs_tests.rs | 3 +- .../src/store/test_support.rs | 5 +- crates/tinyflows-module/Cargo.toml | 8 +- .../src/catalog_legacy_batch.rs | 322 +++++++++++++++ .../src/catalog_legacy_checkpoint.rs | 209 ++++++++++ .../src/catalog_legacy_import.rs | 2 +- crates/tinyflows-module/src/lib.rs | 2 + .../src/run_manager/dedup_settlement_tests.rs | 2 +- .../events_persistence_test_support.rs | 2 +- .../src/run_manager_actor_recovery_tests.rs | 2 +- .../tinyflows-module/src/run_manager_tests.rs | 4 +- .../src/schedule_documents_tests.rs | 2 +- .../src/schedule_members_tests.rs | 2 +- .../src/schedule_service_tests.rs | 4 +- .../tests/module_artifact_tests.rs | 2 +- .../support/manual_worker_artifact_tests.rs | 2 +- .../storage_migration_artifact_tests.rs | 4 +- crates/tinyflows-storage/Cargo.toml | 6 +- crates/tinyflows-storage/src/blocking.rs | 10 +- .../tinyflows-storage/src/blocking_tests.rs | 47 +++ crates/tinyflows-storage/src/key.rs | 6 +- crates/tinyflows-storage/src/key_tests.rs | 22 + crates/tinyflows-storage/src/lib.rs | 4 + .../src/primitives/counters_tests.rs | 36 ++ .../src/primitives/kv_tests.rs | 55 +++ .../src/primitives/leases.rs | 4 +- .../src/primitives/leases_tests.rs | 42 ++ .../tinyflows-storage/src/primitives/repo.rs | 8 +- .../src/primitives/repo_tests.rs | 56 +++ crates/tinyflows-storage/src/schema.rs | 391 ++++++++++++++---- crates/tinyflows-storage/src/schema_tests.rs | 164 ++++++++ crates/tinyflows/Cargo.toml | 2 +- scripts/check-storage-engine-free.sh | 43 ++ 82 files changed, 1648 insertions(+), 381 deletions(-) create mode 100644 crates/tinyflows-module/src/catalog_legacy_batch.rs create mode 100644 crates/tinyflows-module/src/catalog_legacy_checkpoint.rs create mode 100644 crates/tinyflows-storage/src/blocking_tests.rs create mode 100644 crates/tinyflows-storage/src/key_tests.rs create mode 100644 crates/tinyflows-storage/src/primitives/counters_tests.rs create mode 100644 crates/tinyflows-storage/src/primitives/kv_tests.rs create mode 100644 crates/tinyflows-storage/src/primitives/leases_tests.rs create mode 100644 crates/tinyflows-storage/src/primitives/repo_tests.rs create mode 100644 crates/tinyflows-storage/src/schema_tests.rs create mode 100755 scripts/check-storage-engine-free.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7959811..33dcc84a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,19 @@ permissions: contents: read jobs: + msrv: + name: Minimum supported Rust + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@1.88.0 + + - name: Check the workspace at its declared minimum compiler version + run: cargo check --locked --workspace --all-features + rust-sdk: name: Rust SDK runs-on: ubuntu-latest @@ -36,6 +49,9 @@ jobs: - name: Check the published crate carries no storage git dependency run: scripts/check-published-crate.sh + - name: Check storage port users do not link engine crates + run: scripts/check-storage-engine-free.sh + - name: Clippy run: cargo clippy --all-targets -- -D warnings @@ -45,29 +61,6 @@ jobs: - name: Clippy the schedule-only drivers build run: cargo clippy -p tinyflows-drivers --all-targets --no-default-features --features schedule -- -D warnings - - name: Clippy the schedule-only importer build - run: cargo clippy -p tinyflows-import --all-targets --no-default-features --features schedule -- -D warnings - - - name: Clippy the schedule-read importer build - run: cargo clippy -p tinyflows-import --all-targets --no-default-features --features schedule-read -- -D warnings - - - name: Check the schedule-read importer has no driver dependency - run: | - cargo check -p tinyflows-import --no-default-features --features schedule-read - schedule_read_tree=$(cargo tree -p tinyflows-import --no-default-features --features schedule-read -e normal --prefix none --format '{p} features=[{f}]') - if grep -q '^tinyflows-drivers ' <<<"$schedule_read_tree"; then - echo "schedule-read unexpectedly links tinyflows-drivers" >&2 - exit 1 - fi - - - name: Check the schedule importer enables its driver dependency - run: | - schedule_tree=$(cargo tree -p tinyflows-import --no-default-features --features schedule -e normal --prefix none --format '{p} features=[{f}]') - grep -q '^tinyflows-drivers v.* features=\[schedule\]$' <<<"$schedule_tree" - - - name: Check legacy flow and checkpoint importer feature boundaries - run: scripts/check-legacy-import-modes.sh - - name: Build run: cargo build --all-targets @@ -87,14 +80,6 @@ jobs: # The cron store must build and pass without the engine. run: cargo test -p tinyflows-drivers --no-default-features --features schedule - - name: Test the schedule-only importer build - # Legacy cron jobs must import without the flows engine. - run: cargo test -p tinyflows-import --no-default-features --features schedule - - - name: Test the schedule-read importer build - # The bus DTO reader compiles without linking the scheduler or driver implementations. - run: cargo test -p tinyflows-import --no-default-features --features schedule-read - - uses: taiki-e/install-action@cargo-llvm-cov - name: Coverage diff --git a/Cargo.lock b/Cargo.lock index b8bf065b..dd462ecc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -57,9 +57,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" dependencies = [ "aws-lc-sys", "zeroize", @@ -67,9 +67,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" dependencies = [ "cc", "cmake", @@ -148,7 +148,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -188,9 +188,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.5.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df" dependencies = [ "find-msvc-tools", "jobserver", @@ -254,9 +254,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" dependencies = [ "bytes", "memchr", @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -368,7 +368,7 @@ checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -400,7 +400,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -417,9 +417,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -470,9 +470,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "flate2" @@ -547,7 +547,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -720,13 +720,12 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", - "futures-channel", "futures-core", "http", "http-body", @@ -1055,9 +1054,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libsqlite3-sys" @@ -1122,9 +1121,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -1615,9 +1614,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -1631,9 +1630,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -1680,9 +1679,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -1725,9 +1724,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" dependencies = [ "windows-sys 0.61.2", ] @@ -1794,14 +1793,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -1832,9 +1831,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures 0.3.1", @@ -1893,9 +1892,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" [[package]] name = "socket2" @@ -1944,9 +1943,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.6" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -1970,7 +1969,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2014,7 +2013,7 @@ checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2102,7 +2101,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "thiserror", "tokio", @@ -2145,7 +2144,7 @@ dependencies = [ "hex", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tinyflows", "tokio-util", "tracing", @@ -2160,7 +2159,7 @@ dependencies = [ "chrono", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tinyflows", "tinyflows-catalog", "tinytools", @@ -2177,7 +2176,7 @@ dependencies = [ "chrono", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tinyflows", "tinyflows-catalog", "tinyflows-schedule", @@ -2199,7 +2198,7 @@ dependencies = [ "hex", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tempfile", "tinyflows", "tinyflows-bus", @@ -2266,7 +2265,7 @@ dependencies = [ "data-encoding", "serde", "serde_json", - "sha2 0.11.0", + "sha2 0.11.1", "tinystoragedrivers-api", "tinystoragedrivers-testkit", "tinystoragedrivers-types", @@ -2276,8 +2275,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-api" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "bytes", @@ -2290,8 +2289,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-blob" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "bytes", @@ -2304,8 +2303,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-bus" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "serde", "serde_json", @@ -2315,8 +2314,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-busproxy" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "bytes", @@ -2333,8 +2332,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-connect" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "rusqlite", "tinystoragedrivers-types", @@ -2359,8 +2358,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-document" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "data-encoding", @@ -2377,8 +2376,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-kv" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "serde", @@ -2389,8 +2388,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-queue" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "data-encoding", @@ -2405,8 +2404,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-secrets" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "tinystoragedrivers-api", @@ -2429,8 +2428,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-testkit" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "async-trait", "bytes", @@ -2450,8 +2449,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-types" -version = "1.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0-alpha.1#cd89b83c9da55228980cd195bd06d4a851753d10" +version = "2.0.0" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" dependencies = [ "serde", "serde_json", @@ -2487,9 +2486,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" dependencies = [ "bytes", "libc", @@ -2509,7 +2508,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] @@ -2524,9 +2523,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -2698,9 +2697,9 @@ checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] name = "unicode-ident" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "untrusted" @@ -2727,9 +2726,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +checksum = "841583fc36089d7d59419568ae157f43a5a03a2e5a81ba87c77d1596ef46e922" dependencies = [ "base64", "http", @@ -2806,9 +2805,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -2871,7 +2870,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "wasm-bindgen-shared", ] @@ -2906,18 +2905,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -3131,7 +3130,7 @@ checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] @@ -3172,15 +3171,15 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", "synstructure", ] [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" [[package]] name = "zerotrie" @@ -3212,23 +3211,21 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn 3.0.9", ] [[package]] name = "zip" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" dependencies = [ "arbitrary", "crc32fast", "crossbeam-utils", - "displaydoc", "flate2", "indexmap", "memchr", - "thiserror", "zopfli", ] @@ -3246,9 +3243,9 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" dependencies = [ "bumpalo", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index 657d4afd..977547de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ tinybus-module = { git = "https://github.com/tinyhumansai/tinybus", rev = "eb5cf [workspace.package] edition = "2024" -rust-version = "1.85" +rust-version = "1.88" license = "GPL-3.0-or-later" repository = "https://github.com/tinyhumansai/tinyflows" homepage = "https://github.com/tinyhumansai/tinyflows" diff --git a/crates/tinyflows-adaptive/Cargo.toml b/crates/tinyflows-adaptive/Cargo.toml index 0b801ded..4cc6ccee 100644 --- a/crates/tinyflows-adaptive/Cargo.toml +++ b/crates/tinyflows-adaptive/Cargo.toml @@ -2,11 +2,11 @@ name = "tinyflows-adaptive" version = "0.1.0" edition = "2024" -rust-version = "1.85" +rust-version = "1.88" license = "GPL-3.0-or-later" description = "An adaptive loop over the tinyflows engine: select or author a workflow, run it, judge it, learn." -# Names `tinystoragedrivers-core` by git tag, which crates.io cannot resolve -# (docs/storage-crate-split.md); never published. +# Source the public storage ports and TinyFlows-owned helpers from the host's +# storage alpha tag; the host chooses the engine. publish = false [dependencies] @@ -19,8 +19,8 @@ thiserror = "2" # tag so a host that vendors tinystoragedrivers once can `[patch]` it; chrono # turns the RFC 3339 instants of the records into the stored `*_ms`; tokio's # `OnceCell` for one-time collection setup. -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1", optional = true } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1", optional = true } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0", optional = true } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0", optional = true } tinyflows-storage = { path = "../tinyflows-storage", optional = true } chrono = { version = "0.4", default-features = false, features = ["std"], optional = true } tracing = { version = "0.1", default-features = false, features = ["std"], optional = true } @@ -41,9 +41,9 @@ tinyflows = { path = "../tinyflows", features = ["store", "mock"] } # The engine's `WorkflowStore` on an in-memory storage backend, for tests and # the eval example. tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["store"] } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [lints.rust] diff --git a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs index 1b3f1542..d839d8ec 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::drivers::key; use crate::execute::StepRecord; use crate::ledger::{Ledger, Score, conformance}; @@ -9,7 +9,7 @@ use tinyflows_storage::{ Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .unwrap() .documents(), ) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index d47651fc..0137effd 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -1,7 +1,6 @@ //! The adaptive stores' schema v2 declarations, their v1 -> v2 steps and the -//! counter importer. Owner `flows` (scope class Tenant): an owner is declared -//! once, so a host passes `tinyflows_drivers::storage_schema()` through -//! [`extend_storage_schema`]. +//! counter importer. The adaptive crate declares and migrates its own records +//! under owner `flows`, preserving the existing `flows.adaptive_*` namespace. //! //! | v1 | v2 | //! |---|---| @@ -22,24 +21,25 @@ use serde_json::{Map, Value}; use std::sync::Arc; use tinyflows_storage::{ - Cursor, DocumentStore, Filter, ID_FIELD, Key, Migrations, Query, Result, Schema, ScopeClass, - Sort, SourceRecord, Step, TargetRecord, primitives, + Cursor, DocumentStore, Filter, ID_FIELD, Key, Migrations, Owner, Query, Result, Schema, + ScopeClass, Sort, SourceRecord, Step, TargetRecord, primitives, }; use super::convert::{episode_body, lesson_body, row_body, step_body, workflow_body}; use super::{ COUNTER_OWNER, COUNTERS, EPISODES, EVIDENCE, LESSONS, ROWS, SCHEMA_VERSION, SCORES, STEPS, - VARIANTS, WORKFLOWS, collections, + VARIANTS, WORKFLOWS, indexed_collection_specs, }; const COUNTERS_STEP: &str = "adaptive_counters_import"; const PAGE: usize = 200; -/// Declares the adaptive collections on `schema` (owner `flows`) with their -/// v1 -> v2 steps. +/// Adaptive-owned `flows.adaptive_*` declarations and their v1 -> v2 steps. #[must_use] -pub fn extend_storage_schema(mut schema: Schema) -> Schema { - for spec in collections() { +#[allow(clippy::expect_used, reason = "flows is a fixed valid owner name")] +pub fn storage_schema() -> Schema { + let mut schema = Schema::new(Owner::new("flows").expect("valid flows owner")); + for spec in indexed_collection_specs() { schema = schema.collection_with(SCHEMA_VERSION, spec, ScopeClass::Tenant); } schema @@ -53,6 +53,12 @@ pub fn extend_storage_schema(mut schema: Schema) -> Schema { .step(step("adaptive_workflows_v2", WORKFLOWS, workflow)) } +/// Pure adaptive collection declarations for host bootstrap and documentation. +#[must_use] +pub fn collections() -> Vec { + storage_schema().collections() +} + type Convert = fn(&SourceRecord) -> Option; /// A step from the v1 collection named like the v2 entity without its owner @@ -218,7 +224,7 @@ fn workflow(source: &SourceRecord) -> Option { /// v1 stopped. Returns how many sequences were carried. /// /// `tsd.counters` belongs to `tsd`, so a document step cannot write it: the -/// host calls this once per scope after `Registry::migrate`. A completed +/// host calls this once per scope after `storage_schema().migrate`. A completed /// import is skipped; with nothing to carry it only records completion. /// /// # Errors @@ -226,12 +232,11 @@ fn workflow(source: &SourceRecord) -> Option { /// The first storage error; the import resumes on the next call because /// nothing is removed before the counters are raised. pub async fn import_legacy_adaptive_counters(docs: &Arc) -> Result { - let migrations = Migrations::new(docs, "adaptive"); + let migrations = Migrations::new(docs, COUNTER_OWNER); if migrations.done(COUNTER_OWNER, COUNTERS_STEP).await? { return Ok(0); } primitives::ensure(docs).await?; - docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("adaptive.migrations")).await?; docs.ensure_collection(&tinyflows_storage::CollectionSpec::new(COUNTERS)) .await?; let counters = tinyflows_storage::Counters::new(docs); diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index 698736a2..ee3f73cb 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::{Value, json}; use tinyflows_storage::{ - Counters, DocumentStore, DocumentStoreExt, Key, MigrateOptions, Owner, - Precondition, Registry, Schema, Scope, ScopeClass, Storage, StorageBackend, + Counters, DocumentStore, DocumentStoreExt, Key, MigrateOptions, Precondition, Scope, + ScopeClass, Storage, StorageBackend, }; use super::*; @@ -13,15 +13,17 @@ use crate::ledger::{Ledger, conformance}; use crate::workflows::Vault; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::local()).unwrap() + MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap() } fn docs(storage: &Storage) -> Arc { Arc::clone(storage.documents()) } -fn schema() -> Schema { - extend_storage_schema(Schema::new(Owner::new("flows").unwrap())) +fn schema() -> tinyflows_storage::Schema { + storage_schema() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -33,9 +35,8 @@ async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { } async fn migrate(storage: &Storage) { - Registry::compose([schema()]) - .unwrap() - .migrate(storage, &MigrateOptions::new()) + schema() + .migrate(storage.documents(), &MigrateOptions::new()) .await .unwrap(); } @@ -43,7 +44,11 @@ async fn migrate(storage: &Storage) { #[test] fn the_schema_declares_the_eight_adaptive_collections_and_the_vault() { let schema = schema(); - let names: Vec<&str> = schema.items.iter().map(|item| item.name()).collect(); + let names: Vec = schema + .items + .iter() + .map(|(item, _, _)| item.name.clone()) + .collect(); for name in [ "flows.adaptive_rows", "flows.adaptive_lessons", @@ -56,8 +61,18 @@ fn the_schema_declares_the_eight_adaptive_collections_and_the_vault() { ] { assert!(names.contains(&name), "{name} missing from {names:?}"); } - assert!(schema.items.iter().all(|i| i.scope() == ScopeClass::Tenant)); - Registry::compose([schema]).unwrap(); + assert!( + schema + .items + .iter() + .all(|(_, _, scope)| *scope == ScopeClass::Tenant) + ); + assert!( + schema + .collections() + .iter() + .any(|item| item.name == "tsd.migrations") + ); } #[tokio::test] @@ -221,10 +236,7 @@ async fn migrating_twice_changes_nothing() { migrate(&storage).await; migrate(&storage).await; let all = docs(&storage) - .query_all( - "flows.adaptive_variants", - &tinyflows_storage::Query::all(), - ) + .query_all("flows.adaptive_variants", &tinyflows_storage::Query::all()) .await .unwrap(); assert_eq!(all.len(), 1); diff --git a/crates/tinyflows-adaptive/src/drivers/mod.rs b/crates/tinyflows-adaptive/src/drivers/mod.rs index 060604e1..169b9895 100644 --- a/crates/tinyflows-adaptive/src/drivers/mod.rs +++ b/crates/tinyflows-adaptive/src/drivers/mod.rs @@ -40,7 +40,7 @@ //! increment or share an id. First-write-wins edges (`$setOnInsert` in Mongo) //! are insert-only writes. //! -//! [`extend_storage_schema`] declares the collections and their v1 -> v2 +//! [`storage_schema`] declares the collections and their v1 -> v2 //! steps; [`import_legacy_adaptive_counters`] carries the v1 sequences. mod convert; @@ -51,7 +51,7 @@ mod vault; use tinyflows_storage::{CollectionSpec, IndexSpec, StorageError}; pub use ledger::DriverLedger; -pub use migrate::{extend_storage_schema, import_legacy_adaptive_counters}; +pub use migrate::{collections, import_legacy_adaptive_counters, storage_schema}; pub use vault::DriverVault; /// Compare-and-swap attempts before a contended counter update gives up. @@ -75,7 +75,7 @@ pub(crate) const WORKFLOWS: &str = "flows.adaptive_workflows"; pub(crate) const SCHEMA_VERSION: u32 = 2; /// Every collection the stores use, with its indexes. -pub(crate) fn collections() -> Vec { +pub(crate) fn indexed_collection_specs() -> Vec { vec![ CollectionSpec::new(ROWS).index(IndexSpec::new( "by_episode", diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index 083e264a..7ffc46b7 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::workflows::conformance; use serde_json::json; use tinyflows_storage::DocumentStoreExt; @@ -8,7 +8,7 @@ use tinyflows_storage::{ Scope, StorageBackend}; fn memory_docs() -> Arc { Arc::clone( MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .unwrap() .documents(), ) diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index a3c756c5..c71f72a2 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; #[test] fn memory_has_to_be_asked_for_by_name() { @@ -67,7 +67,7 @@ async fn one_call_scopes_both_halves() { async fn one_document_handle_holds_both_halves_and_survives_reopening() { use tinyflows_storage::{ Scope, StorageBackend}; let scoped = MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .expect("scope"); let docs = scoped.documents().clone(); diff --git a/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs index a96a3d2c..770e99bf 100644 --- a/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs +++ b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs @@ -76,7 +76,7 @@ async fn a_ledger_and_a_vault_of_different_kinds_drive_the_same_loop() { { use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; let scoped = MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .expect("scope"); let ledger = tinyflows_adaptive::drivers::DriverLedger::new(scoped.documents().clone()); drive_mixed(&ledger).await; diff --git a/crates/tinyflows-adaptive/tests/support/driver_store.rs b/crates/tinyflows-adaptive/tests/support/driver_store.rs index 02184cd1..ac31ff1b 100644 --- a/crates/tinyflows-adaptive/tests/support/driver_store.rs +++ b/crates/tinyflows-adaptive/tests/support/driver_store.rs @@ -16,7 +16,7 @@ use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; /// An empty store over its own in-memory backend. pub fn empty() -> DriverWorkflowStore { let scoped = MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .expect("a local scope"); DriverWorkflowStore::new(scoped.documents().clone()) } diff --git a/crates/tinyflows-drivers/Cargo.toml b/crates/tinyflows-drivers/Cargo.toml index 07eb1c56..19ad64de 100644 --- a/crates/tinyflows-drivers/Cargo.toml +++ b/crates/tinyflows-drivers/Cargo.toml @@ -51,8 +51,8 @@ store = [ test-fixtures = ["catalog"] [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } tinyflows-storage = { path = "../tinyflows-storage" } serde = { workspace = true } serde_json = { workspace = true } @@ -72,4 +72,4 @@ uuid = { workspace = true, optional = true } [dev-dependencies] async-trait = { workspace = true } tokio = { workspace = true } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index ad0761ff..530cd184 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::catalog::test_support::{catalog, catalog_in, trigger_graph}; use std::sync::Arc; diff --git a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs index df0bd47e..55cbc662 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::catalog::test_support::{catalog, catalog_in}; fn graph() -> Value { diff --git a/crates/tinyflows-drivers/src/catalog/kv_tests.rs b/crates/tinyflows-drivers/src/catalog/kv_tests.rs index bb2381cc..01a0beab 100644 --- a/crates/tinyflows-drivers/src/catalog/kv_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/kv_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::catalog::test_support::{catalog, catalog_in}; use serde_json::json; diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index 66d25d78..c3dbacd9 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -1,14 +1,14 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::catalog::FlowCatalogDocuments; use serde_json::json; use tinyflows_storage::{ - DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, Scope, + DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, Storage, StorageBackend, WriteOptions, }; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::local()).unwrap() + MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -40,9 +40,8 @@ async fn seed_with_created_ms( } async fn migrate(storage: &Storage) { - let registry = Registry::compose([crate::catalog_storage_schema()]).unwrap(); - registry - .migrate(storage, &MigrateOptions::new()) + crate::catalog_storage_schema() + .migrate(storage.documents(), &MigrateOptions::new()) .await .unwrap(); } @@ -55,7 +54,7 @@ fn graph() -> Value { fn the_schema_composes_and_names_every_catalog_collection() { let schema = crate::catalog_storage_schema(); assert_eq!(schema.owner.as_str(), "flows"); - let names: Vec<&str> = schema.items.iter().map(|item| item.name()).collect(); + let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); for name in [ "flows.definitions", "flows.revisions", @@ -63,16 +62,16 @@ fn the_schema_composes_and_names_every_catalog_collection() { "flows.suggestions", "flows.drafts", ] { - assert!(names.contains(&name), "{name}"); + assert!(names.iter().any(|actual| actual == name), "{name}"); } - assert!(schema.items.iter().all(|i| i.scope() == ScopeClass::Tenant)); + assert!(schema.items.iter().all(|(_, _, scope)| *scope == ScopeClass::Tenant)); assert!( !schema .items .iter() - .any(|item| item.name() == "flows.workflows") + .any(|(item, _, _)| item.name == "flows.workflows") ); - Registry::compose([schema]).unwrap(); + assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); } #[tokio::test] diff --git a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs index abc8fb54..5565804e 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use std::sync::Arc; use tinyflows_storage::{DocumentStore, Precondition}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs index f4ddfa36..da1ff29e 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs @@ -1,6 +1,6 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A run lifecycle driven through two catalog handles on one backend. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::test_support::trigger_graph; use tinyflows::nodes::control_flow::dedup::{committed_key, tentative_key}; @@ -195,3 +195,4 @@ async fn parked_finish_does_not_settle_a_run_claimed_for_resume() { "running" ); } + diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs index 194429aa..ddb09283 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs @@ -1,12 +1,12 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use tinyflows_catalog::FlowRunStep; use tinyflows_storage::{ StorageBackend}; async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); let scoped = storage - .for_scope(&tinyflows_storage::Scope::local()) + .for_scope(&tinyflows_storage::Scope::new("local").expect("valid scope")) .unwrap(); let catalog = FlowCatalogDocuments::new(scoped.documents().clone()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ @@ -33,7 +33,7 @@ async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { async fn legacy_parked() -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); let scoped = storage - .for_scope(&tinyflows_storage::Scope::local()) + .for_scope(&tinyflows_storage::Scope::new("local").expect("valid scope")) .unwrap(); let catalog = FlowCatalogDocuments::new(scoped.documents().clone()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ diff --git a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs index 1faff159..301e3281 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use serde_json::json; use std::sync::Arc; use tinyflows_storage::{DocumentStore, Precondition}; diff --git a/crates/tinyflows-drivers/src/catalog/test_support.rs b/crates/tinyflows-drivers/src/catalog/test_support.rs index 72c38c72..0f972c16 100644 --- a/crates/tinyflows-drivers/src/catalog/test_support.rs +++ b/crates/tinyflows-drivers/src/catalog/test_support.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures shared by the catalog tests. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows::model::{Node, NodeKind, WorkflowGraph}; -use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; use super::FlowCatalogDocuments; @@ -208,3 +208,4 @@ impl DocumentStore for Interposed { self.inner.drop_collection(collection).await } } + diff --git a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs index 22a4d252..8d653d55 100644 --- a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use serde_json::json; use std::sync::Arc; use tinyflows::graph::ids::NodeId; diff --git a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs index 82cbbdc9..e9ef7490 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Pins the checkpointer's documents to the layout `tinyagents-graph` stores //! for namespace `flows`. The fixtures are copied from tinyagents' own tests //! (`checkpoint/drivers_tests.rs`, `storage/mod_tests.rs` on its epic branch) //! and from the document shape its `checkpoints_flows_v2` step produces, so a //! drift on either side fails here. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::checkpoint::DriverCheckpointer; use serde_json::json; use std::sync::Arc; @@ -14,7 +14,7 @@ use tinyflows_storage::{DocumentStore, Precondition, Scope, StorageBackend}; fn docs() -> Arc { Arc::clone( MemoryStorage::new() - .for_scope(&Scope::local()) + .for_scope(&Scope::new("local").expect("valid scope")) .unwrap() .documents(), ) @@ -173,3 +173,4 @@ async fn written_documents_have_exactly_the_tinyagents_fields_and_ids() { assert_eq!(writes.doc["checkpoint_id"], "c1"); assert_eq!(writes.doc["ns"], "flows"); } + diff --git a/crates/tinyflows-drivers/src/checkpoint_tests.rs b/crates/tinyflows-drivers/src/checkpoint_tests.rs index a2fec36c..435c3d89 100644 --- a/crates/tinyflows-drivers/src/checkpoint_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_tests.rs @@ -1,15 +1,15 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Behaviour of the storage-driver checkpointer, run against a driver document //! store: the checkpointer contract every durable backend is held to, plus the //! isolation and key-encoding guarantees the driver layout adds. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::checkpoint_layout::{checkpoint_id_key, subgraph_key, thread_id_key}; use serde_json::json; use tinyflows::graph::ids::NodeId; use tinyflows_storage::{ Precondition, Scope, StorageBackend, StorageError}; -fn docs(storage: & scope: &str) -> Arc { +fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( storage .for_scope(&Scope::new(scope).unwrap()) @@ -530,3 +530,4 @@ async fn get_thread_reads_in_insertion_order() { assert_eq!(records[1].state, json!(2)); assert!(cp.get_thread("missing").await.unwrap().is_empty()); } + diff --git a/crates/tinyflows-drivers/src/schedule/admission_tests.rs b/crates/tinyflows-drivers/src/schedule/admission_tests.rs index 7fe87a39..5514b192 100644 --- a/crates/tinyflows-drivers/src/schedule/admission_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/admission_tests.rs @@ -1,5 +1,5 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Shared admission policy fences both configured and existing handles. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, @@ -324,3 +324,4 @@ async fn policy_installation_reconciles_legacy_rows_written_before_adoption() { a.remove_job(&legacy.id).await.unwrap(); a.add_shell_job(None, daily(), "new slot").await.unwrap(); } + diff --git a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs index d626ab16..9ca78e4e 100644 --- a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs @@ -1,5 +1,5 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A completed receipt removes only its own unchanged incarnation. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, @@ -139,3 +139,4 @@ async fn later_acquisition_or_edit_invalidates_the_completed_receipt() { assert!(!a.remove_completed_claim(&fresh).await.unwrap()); assert_eq!(a.list_jobs().await.unwrap().len(), 1); } + diff --git a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs index 64a0f2e4..de40c2d4 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs @@ -1,6 +1,6 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Occurrence leases are compacted once the job has moved past them, without //! weakening the fencing of stale workers. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{daily, docs_in, store}; use super::*; use chrono::{Duration, Utc}; @@ -148,3 +148,4 @@ async fn a_manual_claim_keeps_its_lease_because_the_schedule_did_not_move() { "the occurrence is still ahead of the job, so its lease record stays" ); } + diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index 10bf6be0..4c9882ea 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! A claim's lease and the job document are two writes; these pin down that a //! holder which lost its claim cannot commit through the gap between them, and //! that no error path strands a lease. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::CronDocuments; use super::test_support::{FaultDocuments, daily, docs_in}; use chrono::Duration; @@ -9,7 +9,7 @@ use std::sync::{Arc, atomic::AtomicBool}; use tinyflows_schedule::CronJobPatch; fn fault_store( - backend: & + backend: &MemoryStorage, scope: &str, barrier: Option>, ) -> (CronDocuments, Arc) { @@ -315,3 +315,4 @@ async fn a_failed_lease_release_after_a_committed_completion_does_not_fail_it() assert!(store.list_runs(&job.id, 10).await.unwrap().is_empty()); assert!(store.find_job(&job.id).await.unwrap().is_none()); } + diff --git a/crates/tinyflows-drivers/src/schedule/claims_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_tests.rs index b1591da6..59cb5c91 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_tests.rs @@ -1,5 +1,5 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Occurrences are leased across handles and fenced against stale workers. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, @@ -489,3 +489,4 @@ async fn a_claim_is_a_tsd_lease_and_run_numbers_a_tsd_counter() { .unwrap() ); } + diff --git a/crates/tinyflows-drivers/src/schedule/import_tests.rs b/crates/tinyflows-drivers/src/schedule/import_tests.rs index ecbefd25..875d5e5c 100644 --- a/crates/tinyflows-drivers/src/schedule/import_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/import_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use crate::schedule::job_not_found; use crate::schedule::test_support::{channel_origin, daily, docs_in, store}; diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs index b08b5930..1dd0ecab 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use async_trait::async_trait; use chrono::{Duration, Utc}; use serde_json::Value; @@ -140,7 +140,7 @@ impl DocumentStore for HeldClaimWrite { #[tokio::test] async fn admission_fence_rejects_claim_transaction_that_commits_after_reconciliation() { let backend = MemoryStorage::new(); - let scoped = backend.for_scope(&Scope::local()).unwrap(); + let scoped = backend.for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let held_docs = Arc::new(HeldClaimWrite { inner: scoped.documents().clone(), entered: Notify::new(), @@ -208,7 +208,7 @@ async fn admission_fence_rejects_claim_transaction_that_commits_after_reconcilia #[tokio::test] async fn reconciliation_does_not_fence_changed_snapshots_and_lost_fence_reply_is_idempotent() { let backend = MemoryStorage::new(); - let scoped = backend.for_scope(&Scope::local()).unwrap(); + let scoped = backend.for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let held_docs = Arc::new(HeldClaimWrite { inner: scoped.documents().clone(), entered: Notify::new(), diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index 9a9f8e32..b874ab70 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -18,7 +18,7 @@ //! `tsd.counters` belongs to `tsd`, so a document step cannot write it. The //! step that reads `cron_counters` therefore files the run counter as //! `cron.admission/runs_counter` (`{next}`), and [`import_legacy_cron_counters`] -//! moves it into `tsd.counters` once per scope after `Registry::migrate`. +//! moves it into `tsd.counters` once per scope after this schema migration. use serde_json::{Map, Value, json}; use std::sync::Arc; @@ -35,8 +35,7 @@ const STAGED_COUNTER: &str = "runs_counter"; const COUNTERS_STEP: &str = "counters_import"; /// The `cron` owner's declarations (`cron.jobs`, `cron.runs`, -/// `cron.admission`) and their v1 -> v2 steps. The host composes it with -/// `Registry::compose` and runs `Registry::migrate` per scope. +/// `cron.admission`) and their v1 -> v2 steps. #[must_use] #[allow(clippy::expect_used, reason = "`cron` is a valid owner name")] pub fn storage_schema() -> Schema { @@ -255,7 +254,6 @@ pub async fn import_legacy_cron_counters(docs: &Arc) -> Resul return Ok(false); } tinyflows_storage::primitives::ensure(docs).await?; - docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("cron.migrations")).await?; docs.ensure_collection(&CronDocuments::collections().remove(2)) .await?; let mut carried = false; diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index 35f08583..bf2cd988 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -1,15 +1,15 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use chrono::TimeZone; use serde_json::json; use tinyflows_schedule::Schedule; use tinyflows_storage::{ - DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, Scope, + DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, Storage, StorageBackend, }; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::local()).unwrap() + MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -21,9 +21,8 @@ async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { } async fn migrate(storage: &Storage) { - Registry::compose([storage_schema()]) - .unwrap() - .migrate(storage, &MigrateOptions::new()) + storage_schema() + .migrate(storage.documents(), &MigrateOptions::new()) .await .unwrap(); } @@ -47,9 +46,9 @@ fn v1_job(schedule: Value) -> Value { fn the_schema_composes_and_declares_the_cron_collections() { let schema = storage_schema(); assert_eq!(schema.owner.as_str(), "cron"); - let names: Vec<&str> = schema.items.iter().map(|item| item.name()).collect(); + let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); assert_eq!(names, ["cron.jobs", "cron.runs", "cron.admission"]); - Registry::compose([schema]).unwrap(); + assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); } #[tokio::test] @@ -89,9 +88,9 @@ async fn v1_jobs_runs_and_the_ledger_migrate_and_read_back() { seed(&storage, "cron_counters", "runs", json!({"next": 8})).await; migrate(&storage).await; - assert!(import_legacy_cron_counters(&storage).await.unwrap()); + assert!(import_legacy_cron_counters(storage.documents()).await.unwrap()); assert!( - !import_legacy_cron_counters(&storage).await.unwrap(), + !import_legacy_cron_counters(storage.documents()).await.unwrap(), "runs once" ); diff --git a/crates/tinyflows-drivers/src/schedule/mod_tests.rs b/crates/tinyflows-drivers/src/schedule/mod_tests.rs index 60e203ff..dae72496 100644 --- a/crates/tinyflows-drivers/src/schedule/mod_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/mod_tests.rs @@ -1,5 +1,5 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use super::test_support::{daily, docs_in, store}; use chrono::Utc; diff --git a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs index 6401916a..d4cdf2ab 100644 --- a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs @@ -1,5 +1,5 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Removal compatibility and admission bookkeeping across supported backends. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::super::{ CronDocuments, JOBS, test_support::{FaultDocuments, daily, docs_in}, @@ -135,3 +135,4 @@ async fn a_legacy_row_without_enabled_is_imported_as_disabled() { b.add_shell_job(None, daily(), "enabled").await.unwrap(); assert_eq!(b.list_jobs().await.unwrap().len(), 2); } + diff --git a/crates/tinyflows-drivers/src/schedule/test_support.rs b/crates/tinyflows-drivers/src/schedule/test_support.rs index 9d717215..e5187cf8 100644 --- a/crates/tinyflows-drivers/src/schedule/test_support.rs +++ b/crates/tinyflows-drivers/src/schedule/test_support.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures the schedule tests share. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows_schedule::{JobOrigin, Schedule}; -use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; use super::CronDocuments; @@ -205,3 +205,4 @@ impl DocumentStore for FaultDocuments { self.inner.atomic_batch(ops).await } } + diff --git a/crates/tinyflows-drivers/src/state_store_tests.rs b/crates/tinyflows-drivers/src/state_store_tests.rs index 61e5afbb..0343dd6a 100644 --- a/crates/tinyflows-drivers/src/state_store_tests.rs +++ b/crates/tinyflows-drivers/src/state_store_tests.rs @@ -1,9 +1,9 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use serde_json::json; use tinyflows_storage::{ Scope, StorageBackend}; -fn docs(storage: & scope: &str) -> Arc { +fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( storage .for_scope(&Scope::new(scope).unwrap()) diff --git a/crates/tinyflows-drivers/src/storage/kv_import.rs b/crates/tinyflows-drivers/src/storage/kv_import.rs index b5633dce..31e7e963 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import.rs @@ -2,7 +2,7 @@ //! //! A document step may only write its own owner's collections and `tsd.kv` //! belongs to `tsd`, so the move is an importer the host calls once per scope -//! after `Registry::migrate` (the pattern `tinyagents-harness` uses): +//! after the driver's own schema migration: //! [`import_legacy_kv`] copies `flows_kv` (the catalog's namespaced state, //! `{namespace, key, value_json}`) and `flows_state` (`DriverStateStore`, //! `{key, value}`), removes them and records completion in `tsd.migrations`. @@ -111,7 +111,6 @@ pub async fn import_legacy_kv(docs: &Arc) -> Result { return Ok(0); } primitives::ensure(docs).await?; - docs.ensure_collection(&CollectionSpec::new("flows.migrations")).await?; let written = copy(docs, "flows_kv", convert_catalog).await? + copy(docs, "flows_state", convert_state).await?; migrations.mark_done(KV_OWNER, STEP).await?; diff --git a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs index cd67a128..1db945bc 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs @@ -1,10 +1,10 @@ -use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use super::*; use serde_json::json; -use tinyflows_storage::{Kv, Scope, StorageBackend}; +use tinyflows_storage::{Kv, Scope, Storage, StorageBackend}; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::local()).unwrap() + MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -41,7 +41,7 @@ async fn both_v1_collections_move_into_tsd_kv_with_values_unwrapped() { ) .await; - assert_eq!(import_legacy_kv(&storage).await.unwrap(), 2); + assert_eq!(import_legacy_kv(storage.documents()).await.unwrap(), 2); let catalog = Kv::ns(storage.documents(), "flows", "kv:ns").unwrap(); assert_eq!( @@ -81,13 +81,13 @@ async fn the_import_runs_once_and_keeps_newer_v2_values() { let state = Kv::ns(storage.documents(), "flows", "state").unwrap(); state.put("k", &"new").await.unwrap(); - assert_eq!(import_legacy_kv(&storage).await.unwrap(), 0); + assert_eq!(import_legacy_kv(storage.documents()).await.unwrap(), 0); assert_eq!( state.get::("k").await.unwrap().as_deref(), Some("new") ); assert!( - Migrations::new(&storage) + Migrations::new(storage.documents(), KV_OWNER) .done("flows", "kv_import") .await .unwrap() @@ -101,7 +101,7 @@ async fn the_import_runs_once_and_keeps_newer_v2_values() { ) .await; assert_eq!( - import_legacy_kv(&storage).await.unwrap(), + import_legacy_kv(storage.documents()).await.unwrap(), 0, "a finished import is not repeated" ); diff --git a/crates/tinyflows-drivers/src/storage/mod.rs b/crates/tinyflows-drivers/src/storage/mod.rs index 754e9f64..e6523692 100644 --- a/crates/tinyflows-drivers/src/storage/mod.rs +++ b/crates/tinyflows-drivers/src/storage/mod.rs @@ -15,14 +15,14 @@ //! - Key-value state and counters live in `tsd.kv` / `tsd.counters`; a document //! step cannot write another owner's collections, so [`import_legacy_kv`] and //! [`import_legacy_cron_counters`] move them (host calls them once per scope -//! after `Registry::migrate`). +//! after the cron schema migration. //! -//! A host composes `Registry::compose([storage_schema(), cron_storage_schema(), -//! ...])` and runs `Registry::migrate` per scope. +//! Each owner runs its own schema migration against the scoped document handle +//! it is given; hosts use [`collections()`] for backend bootstrap. mod kv_import; -use tinyflows_storage::{CollectionSpec, Owner, Schema, ScopeClass}; +use tinyflows_storage::{CollectionSpec, Owner, Schema}; pub use kv_import::{ CATALOG_NAMESPACE_PREFIX, KV_OWNER, STATE_NAMESPACE, catalog_namespace, import_legacy_kv, @@ -38,9 +38,8 @@ fn flows_schema() -> Schema { } /// The `flows` owner's declarations and v1 -> v2 steps for the features this -/// build has. `tinyflows-adaptive` extends the result with its collections -/// (`tinyflows_adaptive::drivers::extend_storage_schema`): an owner may be -/// declared only once, so the host passes one schema through both. +/// build has. Adaptive declares and runs its own schema for its separate +/// `flows.adaptive_*` collections. #[must_use] pub fn storage_schema() -> Schema { #[allow(unused_mut)] diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index 81a23f7a..66dfd7d0 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -1,9 +1,9 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Tests for patch-based workflow editing. //! //! The theme is that a bad edit costs an error message, never the saved //! workflow. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use serde_json::json; @@ -32,7 +32,7 @@ fn document(id: &str) -> String { .to_string() } -fn store() -> ( Arc) { +fn store() -> (MemoryStorage, Arc) { let storage = MemoryStorage::new(); let store: Arc = Arc::new(store_over(&storage, "local")); (storage, store) @@ -322,3 +322,4 @@ fn editing_a_workflow_that_does_not_exist_says_so() { assert!(matches!(err, WorkflowError::NotFound(_)), "got {err:?}"); } + diff --git a/crates/tinyflows-drivers/src/store/concurrency_tests.rs b/crates/tinyflows-drivers/src/store/concurrency_tests.rs index b4144f86..8bbf416f 100644 --- a/crates/tinyflows-drivers/src/store/concurrency_tests.rs +++ b/crates/tinyflows-drivers/src/store/concurrency_tests.rs @@ -1,8 +1,8 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Racing writers: the compare-and-swap on a definition and the revision //! captured beside it must hold up when two threads, or two store instances, //! write at once. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::{Arc, Barrier}; use tinyflows::store::WorkflowStore; @@ -189,3 +189,4 @@ fn a_guarded_save_of_an_invalid_graph_is_refused_before_anything_is_written() { ); assert!(store.list_revisions("guarded").unwrap().is_empty()); } + diff --git a/crates/tinyflows-drivers/src/store/history_tests.rs b/crates/tinyflows-drivers/src/store/history_tests.rs index d2a81f7b..463ecd40 100644 --- a/crates/tinyflows-drivers/src/store/history_tests.rs +++ b/crates/tinyflows-drivers/src/store/history_tests.rs @@ -1,9 +1,9 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Revision snapshotting, undo, and rollback: saving over a workflow snapshots //! what it replaced, undo restores (and is itself undoable), a named rollback //! restores exactly that revision, and history never leaks into the ordinary //! workflow listing. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{record, store, store_over}; use tinyflows::store::types::WorkflowError; use tinyflows::store::{MAX_REVISIONS, WorkflowStore, require, rollback, undo_last}; @@ -211,3 +211,4 @@ fn history_does_not_show_up_in_the_workflow_listing() { assert_eq!(store.list().unwrap().len(), 1); assert_eq!(require(&store, "greet").unwrap().description, "again"); } + diff --git a/crates/tinyflows-drivers/src/store/locks_tests.rs b/crates/tinyflows-drivers/src/store/locks_tests.rs index eb5a2279..f9079d49 100644 --- a/crates/tinyflows-drivers/src/store/locks_tests.rs +++ b/crates/tinyflows-drivers/src/store/locks_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! The decision lease: exclusion across store instances, release on drop, and //! recovery from a holder that went away. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::mpsc; use std::time::Duration; @@ -146,3 +146,4 @@ fn a_lease_id_that_is_not_usable_is_refused() { let store = store_over(&MemoryStorage::new(), "local"); assert!(store.lock_proposal_decision("").is_err()); } + diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index a1c3089d..6d6fbac4 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -4,14 +4,14 @@ use std::sync::Arc; use serde_json::{Value, json}; use tinyflows::store::WorkflowStore; use tinyflows_storage::{ - DocumentStore, DocumentStoreExt, MigrateOptions, Precondition, Query, Registry, + DocumentStore, DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, ScopeClass, Storage, StorageBackend, }; use super::*; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::local()).unwrap() + MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() } fn docs(storage: &Storage) -> Arc { @@ -27,9 +27,8 @@ async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { } async fn migrate(storage: &Storage) { - Registry::compose([crate::storage_schema()]) - .unwrap() - .migrate(storage, &MigrateOptions::new()) + crate::storage_schema() + .migrate(storage.documents(), &MigrateOptions::new()) .await .unwrap(); } @@ -44,7 +43,7 @@ fn v1_record(id: &str, description: &str) -> String { fn the_schema_names_every_workflow_store_collection() { let schema = crate::storage_schema(); assert_eq!(schema.owner.as_str(), "flows"); - let names: Vec<&str> = schema.items.iter().map(|item| item.name()).collect(); + let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); for name in [ "flows.workflows", "flows.workflow_revisions", @@ -53,10 +52,10 @@ fn the_schema_names_every_workflow_store_collection() { "flows.workflow_proposals", "flows.workflow_quarantine", ] { - assert!(names.contains(&name), "{name} missing from {names:?}"); + assert!(names.iter().any(|actual| actual == name), "{name} missing from {names:?}"); } - assert!(schema.items.iter().all(|i| i.scope() == ScopeClass::Tenant)); - Registry::compose([schema]).unwrap(); + assert!(schema.items.iter().all(|(_, _, scope)| *scope == ScopeClass::Tenant)); + assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); } #[tokio::test] diff --git a/crates/tinyflows-drivers/src/store/mod_tests.rs b/crates/tinyflows-drivers/src/store/mod_tests.rs index bca89c03..4957f3df 100644 --- a/crates/tinyflows-drivers/src/store/mod_tests.rs +++ b/crates/tinyflows-drivers/src/store/mod_tests.rs @@ -1,8 +1,8 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Save/delete round-tripping: the graph and host fields survive a save, the //! `defaults` block round-trips, an uncompilable graph is refused, ids are //! guarded, and the host policy judges reads as well as writes. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use serde_json::json; @@ -337,3 +337,4 @@ fn debug_names_the_store_without_dumping_the_handle() { fn the_journal_cap_is_the_engines() { assert_eq!(JOURNAL_CAP, tinyflows::store::MAX_NOTES); } + diff --git a/crates/tinyflows-drivers/src/store/notes_tests.rs b/crates/tinyflows-drivers/src/store/notes_tests.rs index 58881805..600f7e8b 100644 --- a/crates/tinyflows-drivers/src/store/notes_tests.rs +++ b/crates/tinyflows-drivers/src/store/notes_tests.rs @@ -1,8 +1,8 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! The journal on documents: ordering, scoping, supersession, the cap, and what //! happens when things go wrong: a journal that cannot be parsed, an id that is //! not usable, concurrent appenders. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{NoteKind, NoteSource, WorkflowError, WorkflowNote}; use tinyflows::store::{MAX_NOTES, WorkflowStore, current_notes, mint_note_id}; @@ -287,3 +287,4 @@ fn a_note_round_trips_every_field() { store.append_note(&written).unwrap(); assert_eq!(store.list_notes("sweep").unwrap().remove(0), written); } + diff --git a/crates/tinyflows-drivers/src/store/proposals_tests.rs b/crates/tinyflows-drivers/src/store/proposals_tests.rs index d27815fb..87fc5c93 100644 --- a/crates/tinyflows-drivers/src/store/proposals_tests.rs +++ b/crates/tinyflows-drivers/src/store/proposals_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Proposals on documents. A proposal is the one artifact in this feature that //! can change a saved graph, so what is asserted here is mostly about *not* //! doing that by accident: a listing that cannot leak another workflow's //! proposals, a decision that replaces rather than adds, a save that refuses //! when the graph moved underneath it. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::atomic::Ordering; use serde_json::json; @@ -285,3 +285,4 @@ fn an_edit_that_lands_between_the_check_and_the_write_refuses_the_proposal() { .is_empty() ); } + diff --git a/crates/tinyflows-drivers/src/store/runs_tests.rs b/crates/tinyflows-drivers/src/store/runs_tests.rs index 5b4dc002..b85f71f9 100644 --- a/crates/tinyflows-drivers/src/store/runs_tests.rs +++ b/crates/tinyflows-drivers/src/store/runs_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Run-record listing: newest first, scoped to their workflow, and a run that //! was never recorded is a distinct error rather than a silent `None`. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{block_on, store}; use tinyflows::store::types::{RunExecutor, RunStatus, WorkflowError}; use tinyflows::store::{WorkflowStore, new_run_record, require_run}; @@ -154,3 +154,4 @@ fn an_unreadable_run_document_is_skipped_by_listings_and_named_by_a_direct_read( WorkflowError::Malformed(_) )); } + diff --git a/crates/tinyflows-drivers/src/store/test_support.rs b/crates/tinyflows-drivers/src/store/test_support.rs index 38ab77b9..3967be6b 100644 --- a/crates/tinyflows-drivers/src/store/test_support.rs +++ b/crates/tinyflows-drivers/src/store/test_support.rs @@ -1,12 +1,12 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; //! Fixtures the store tests share. +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use serde_json::json; use tinyflows::store::parse_workflow; use tinyflows::store::types::WorkflowRecord; -use tinyflows_storage::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; use super::DriverWorkflowStore; @@ -241,3 +241,4 @@ impl DocumentStore for Faulty { self.inner.atomic_batch_checked(checks, ops).await } } + diff --git a/crates/tinyflows-module/Cargo.toml b/crates/tinyflows-module/Cargo.toml index b413eb06..e23cae1c 100644 --- a/crates/tinyflows-module/Cargo.toml +++ b/crates/tinyflows-module/Cargo.toml @@ -14,10 +14,10 @@ crate-type = ["rlib", "cdylib"] [dependencies] tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["catalog", "engine", "schedule"] } tinyflows-catalog = { path = "../tinyflows-catalog" } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } tinyflows-storage = { path = "../tinyflows-storage" } -tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } tinyflows-schedule = { path = "../tinyflows-schedule" } anyhow = "1" chrono.workspace = true @@ -43,4 +43,4 @@ static-link = [] [dev-dependencies] tempfile = "3" tokio = { workspace = true, features = ["test-util"] } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } diff --git a/crates/tinyflows-module/src/catalog_legacy_batch.rs b/crates/tinyflows-module/src/catalog_legacy_batch.rs new file mode 100644 index 00000000..67f906c4 --- /dev/null +++ b/crates/tinyflows-module/src/catalog_legacy_batch.rs @@ -0,0 +1,322 @@ +//! Shared semantic writer for legacy flow and checkpoint records. + +use std::collections::{BTreeMap, HashSet}; + +use anyhow::{Context, Result}; +use serde_json::Value; +use tinyflows_bus::catalog::{ + ImportLegacyBatchResponse, LegacyImportKind, LegacyImportProblem, LegacyImportRecord, + LegacyImportTally, +}; +use tinyflows_catalog::{ + Flow, FlowDraft, FlowRevision, FlowRun, FlowRunStep, FlowSuggestion, SuggestionStatus, +}; +use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; + +#[derive(Default)] +struct Counts { + written: u64, + existing: u64, + invalid: u64, +} + +/// Applies one already bounded batch through the caller's scoped catalog and checkpointer. +/// +/// Source parsing stays in the host-only reader; this shared writer owns legacy +/// semantic decoding, migration, idempotent catalog writes and checkpoint replay. +/// The host must run all batches during exclusive migration startup, before +/// run execution or other writers can mutate the target scope. Checkpoints use +/// the append-only sequence protocol and are not atomically fenced from a live +/// checkpointer writer. +pub async fn apply_legacy_batch( + records: &[LegacyImportRecord], + catalog: &FlowCatalogDocuments, + checkpointer: Option<&DriverCheckpointer>, +) -> Result { + let mut counts: BTreeMap = BTreeMap::new(); + let mut problems = Vec::new(); + let mut imported_flows = HashSet::new(); + let mut checkpoint_records = Vec::new(); + + for record in records { + match record { + LegacyImportRecord::Flow(row) => match decode_flow(row) { + Ok(flow) => match catalog.import_flow(&flow).await { + Ok(wrote) => { + tally(&mut counts, LegacyImportKind::Flow, wrote); + imported_flows.insert(flow.id); + } + Err(error) => return Err(error), + }, + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Flow, + Some(&row.id), + error.to_string(), + ), + }, + LegacyImportRecord::Revision(row) => { + let decoded = (|| -> Result { + let graph = + serde_json::from_str(&row.graph_json).context("graph_json is not JSON")?; + Ok(FlowRevision { + id: row.id.clone(), + flow_id: row.flow_id.clone(), + graph, + name: row.name.clone(), + require_approval: row.require_approval, + created_at: row.created_at.clone(), + }) + })(); + match decoded { + Ok(value) => { + if imported_flows.contains(&value.flow_id) + || catalog.get_flow(&value.flow_id).await?.is_some() + { + tally( + &mut counts, + LegacyImportKind::Revision, + catalog.import_revision(&value).await?, + ); + } else { + problem( + &mut counts, + &mut problems, + LegacyImportKind::Revision, + Some(&value.id), + "owner flow was not imported".into(), + ); + } + } + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Revision, + Some(&row.id), + error.to_string(), + ), + } + } + LegacyImportRecord::Run(row) => { + let decoded = (|| -> Result { + Ok(FlowRun { + id: row.id.clone(), + flow_id: row.flow_id.clone(), + thread_id: row.thread_id.clone(), + status: row.status.clone(), + started_at: row.started_at.clone(), + finished_at: row.finished_at.clone(), + steps: serde_json::from_str::>(&row.steps_json) + .context("run steps are corrupt")?, + pending_approvals: serde_json::from_str(&row.pending_approvals_json) + .context("pending approvals are corrupt")?, + error: row.error.clone(), + graph_hash: row.graph_hash.clone(), + }) + })(); + match decoded { + Ok(value) => { + if imported_flows.contains(&value.flow_id) + || catalog.get_flow(&value.flow_id).await?.is_some() + { + tally( + &mut counts, + LegacyImportKind::Run, + catalog.import_run(&value).await?, + ); + } else { + problem( + &mut counts, + &mut problems, + LegacyImportKind::Run, + Some(&value.id), + "owner flow was not imported".into(), + ); + } + } + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Run, + Some(&row.id), + error.to_string(), + ), + } + } + LegacyImportRecord::State(row) => { + match serde_json::from_str::(&row.value_json) { + Ok(value) => tally( + &mut counts, + LegacyImportKind::State, + catalog.import_kv(&row.namespace, &row.key, &value).await?, + ), + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::State, + Some(&row.key), + error.to_string(), + ), + } + } + LegacyImportRecord::Suggestion(row) => { + let decoded = (|| -> Result { + Ok(FlowSuggestion { + id: row.id.clone(), + title: row.title.clone(), + one_liner: row.one_liner.clone(), + rationale: row.rationale.clone(), + trigger_hint: row.trigger_hint.clone(), + steps_outline: serde_json::from_str(&row.steps_json) + .context("suggestion steps are corrupt")?, + suggested_connections: serde_json::from_str(&row.connections_json) + .context("suggestion connections are corrupt")?, + suggested_slugs: serde_json::from_str(&row.slugs_json) + .context("suggestion slugs are corrupt")?, + build_prompt: row.build_prompt.clone(), + confidence: row.confidence, + status: SuggestionStatus::from_str_lossy(&row.status), + created_at: row.created_at.clone(), + source_run_id: row.source_run_id.clone(), + }) + })(); + match decoded { + Ok(value) => tally( + &mut counts, + LegacyImportKind::Suggestion, + catalog.import_suggestion(&value).await?, + ), + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Suggestion, + Some(&row.id), + error.to_string(), + ), + } + } + LegacyImportRecord::Draft(row) => match serde_json::from_slice::(&row.bytes) + { + Ok(value) if safe_draft_id(&value.id) => tally( + &mut counts, + LegacyImportKind::Draft, + catalog.import_draft(&value).await?, + ), + Ok(value) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Draft, + Some(&row.file_name), + format!("'{}' is not a usable draft id", value.id), + ), + Err(error) => problem( + &mut counts, + &mut problems, + LegacyImportKind::Draft, + Some(&row.file_name), + error.to_string(), + ), + }, + LegacyImportRecord::Checkpoint { .. } + | LegacyImportRecord::SqliteCheckpointWrite(_) + | LegacyImportRecord::FileCheckpointWrite { .. } => { + checkpoint_records.push(record.clone()); + } + LegacyImportRecord::Invalid { + record_kind: LegacyImportKind::Checkpoint | LegacyImportKind::Write, + .. + } => { + checkpoint_records.push(record.clone()); + } + LegacyImportRecord::Invalid { + record_kind, + identity, + message, + } => problem( + &mut counts, + &mut problems, + *record_kind, + identity.as_deref(), + message.clone(), + ), + } + } + if !checkpoint_records.is_empty() { + let checkpointer = + checkpointer.context("checkpoint import requires a scoped checkpointer")?; + let response = crate::catalog_legacy_checkpoint::apply_legacy_checkpoint_batch( + &checkpoint_records, + checkpointer, + ) + .await?; + for tally in response.tallies { + let count = counts.entry(tally.kind).or_default(); + count.written += tally.written; + count.existing += tally.existing; + count.invalid += tally.invalid; + } + problems.extend(response.problems); + } + Ok(ImportLegacyBatchResponse { + tallies: counts + .into_iter() + .map(|(kind, count)| LegacyImportTally { + kind, + written: count.written, + existing: count.existing, + invalid: count.invalid, + }) + .collect(), + problems, + }) +} + +fn decode_flow(row: &tinyflows_bus::catalog::LegacyFlowRecord) -> Result { + let raw: Value = serde_json::from_str(&row.graph_json).context("graph_json is not JSON")?; + let migrated = tinyflows::migrate::migrate(raw).map_err(|error| anyhow::anyhow!("{error}"))?; + let graph = serde_json::from_value(migrated).context("graph_json does not decode")?; + Ok(Flow { + id: row.id.clone(), + name: row.name.clone(), + graph, + enabled: row.enabled, + created_at: row.created_at.clone(), + updated_at: row.updated_at.clone(), + last_run_at: row.last_run_at.clone(), + last_status: row.last_status.clone(), + require_approval: row.require_approval, + }) +} + +fn safe_draft_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 64 + && id.chars().all(|character| { + character.is_ascii_alphanumeric() || character == '-' || character == '_' + }) +} + +fn tally(counts: &mut BTreeMap, kind: LegacyImportKind, wrote: bool) { + let count = counts.entry(kind).or_default(); + if wrote { + count.written += 1 + } else { + count.existing += 1 + } +} + +fn problem( + counts: &mut BTreeMap, + problems: &mut Vec, + kind: LegacyImportKind, + identity: Option<&str>, + message: String, +) { + counts.entry(kind).or_default().invalid += 1; + problems.push(LegacyImportProblem { + kind, + identity: identity.map(|value| value.chars().take(128).collect()), + message: message.chars().take(240).collect(), + }); +} diff --git a/crates/tinyflows-module/src/catalog_legacy_checkpoint.rs b/crates/tinyflows-module/src/catalog_legacy_checkpoint.rs new file mode 100644 index 00000000..63994711 --- /dev/null +++ b/crates/tinyflows-module/src/catalog_legacy_checkpoint.rs @@ -0,0 +1,209 @@ +//! Shared semantic importer for checkpoint and pending-write records. + +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use serde::Deserialize; +use serde_json::Value; +use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; +use tinyflows_bus::catalog::{ + ImportLegacyBatchResponse, LegacyImportKind, LegacyImportProblem, LegacyImportRecord, + LegacyImportTally, +}; +use tinyflows_drivers::DriverCheckpointer; + +#[derive(Default)] +struct Counts { + written: u64, + existing: u64, + invalid: u64, +} + +/// Imports raw checkpoint/write DTOs through the shared checkpointer writer. +/// +/// Like the original one-time importer, this requires exclusive migration +/// startup access to the target scope before live run writers start. +pub async fn apply_legacy_checkpoint_batch( + records: &[LegacyImportRecord], + checkpointer: &DriverCheckpointer, +) -> Result { + let mut counts = BTreeMap::::new(); + let mut problems = Vec::new(); + let mut checkpoints = Vec::new(); + let mut writes = Vec::new(); + for record in records { + match record { + LegacyImportRecord::Checkpoint { raw_json } => { + match serde_json::from_str::>(raw_json) { + Ok(checkpoint) => checkpoints.push(checkpoint), + Err(error) => invalid( + &mut counts, + &mut problems, + LegacyImportKind::Checkpoint, + None, + error.to_string(), + ), + } + } + LegacyImportRecord::SqliteCheckpointWrite(row) => { + let decoded = (|| -> Result<_> { + Ok(( + CheckpointConfig { + thread_id: row.thread_id.clone(), + checkpoint_id: Some(row.checkpoint_id.clone()), + namespace: serde_json::from_str(&row.namespace_json) + .context("write namespace is corrupt")?, + }, + PendingWrite { + node: tinyflows::graph::ids::NodeId::new(row.node.clone()), + task_id: row.task_id.clone(), + idx: row.idx, + channel: row.channel.clone(), + payload: serde_json::from_str(&row.payload_json) + .context("write payload is corrupt")?, + }, + )) + })(); + match decoded { + Ok(value) => writes.push(value), + Err(error) => invalid( + &mut counts, + &mut problems, + LegacyImportKind::Write, + Some(&row.task_id), + error.to_string(), + ), + } + } + LegacyImportRecord::FileCheckpointWrite { + thread_id, + raw_json, + } => match serde_json::from_str::(raw_json) { + Ok(row) => writes.push(( + CheckpointConfig { + thread_id: thread_id.clone(), + checkpoint_id: Some(row.checkpoint_id), + namespace: row.namespace, + }, + row.write, + )), + Err(error) => invalid( + &mut counts, + &mut problems, + LegacyImportKind::Write, + Some(thread_id), + error.to_string(), + ), + }, + LegacyImportRecord::Invalid { + record_kind, + identity, + message, + } if matches!( + record_kind, + LegacyImportKind::Checkpoint | LegacyImportKind::Write + ) => + { + invalid( + &mut counts, + &mut problems, + *record_kind, + identity.as_deref(), + message.clone(), + ); + } + _ => { + return Err(anyhow::anyhow!( + "checkpoint batch contains a non-checkpoint record" + )); + } + } + } + replay(checkpointer, checkpoints, writes, &mut counts).await?; + Ok(ImportLegacyBatchResponse { + tallies: counts + .into_iter() + .map(|(kind, count)| LegacyImportTally { + kind, + written: count.written, + existing: count.existing, + invalid: count.invalid, + }) + .collect(), + problems, + }) +} + +#[derive(Deserialize)] +struct FileWrite { + #[serde(default)] + namespace: Vec, + checkpoint_id: String, + write: PendingWrite, +} + +fn invalid( + counts: &mut BTreeMap, + problems: &mut Vec, + kind: LegacyImportKind, + identity: Option<&str>, + message: String, +) { + counts.entry(kind).or_default().invalid += 1; + problems.push(LegacyImportProblem { + kind, + identity: identity.map(|value| value.chars().take(128).collect()), + message: message.chars().take(240).collect(), + }); +} + +async fn replay( + checkpointer: &DriverCheckpointer, + checkpoints: Vec>, + writes: Vec<(CheckpointConfig, PendingWrite)>, + counts: &mut BTreeMap, +) -> Result<()> { + for checkpoint in checkpoints { + let held = checkpointer + .get_scoped( + &checkpoint.thread_id, + Some(&checkpoint.checkpoint_id), + &checkpoint.namespace, + ) + .await + .map_err(|error| anyhow::anyhow!("{error}"))? + .is_some(); + if held { + counts + .entry(LegacyImportKind::Checkpoint) + .or_default() + .existing += 1; + continue; + } + checkpointer + .put(checkpoint) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + counts + .entry(LegacyImportKind::Checkpoint) + .or_default() + .written += 1; + } + let mut groups: Vec<(CheckpointConfig, Vec)> = Vec::new(); + for (config, write) in writes { + match groups.iter_mut().find(|(known, _)| *known == config) { + Some((_, group)) => group.push(write), + None => groups.push((config, vec![write])), + } + } + for (config, group) in groups { + let (written, existing) = checkpointer + .import_writes_if_absent(&config, &group) + .await + .map_err(|error| anyhow::anyhow!("{error}"))?; + let count = counts.entry(LegacyImportKind::Write).or_default(); + count.written += written as u64; + count.existing += existing as u64; + } + Ok(()) +} diff --git a/crates/tinyflows-module/src/catalog_legacy_import.rs b/crates/tinyflows-module/src/catalog_legacy_import.rs index d3a98c4a..d701ad4e 100644 --- a/crates/tinyflows-module/src/catalog_legacy_import.rs +++ b/crates/tinyflows-module/src/catalog_legacy_import.rs @@ -39,7 +39,7 @@ pub(super) async fn import( let docs = Arc::clone(&port) as Arc; let catalog = FlowCatalogDocuments::new(Arc::clone(&docs)); let checkpointer = DriverCheckpointer::::new(docs); - tinyflows_import::apply_legacy_batch(&request.records, &catalog, Some(&checkpointer)) + super::catalog_legacy_batch::apply_legacy_batch(&request.records, &catalog, Some(&checkpointer)) .await .map_err(failure) } diff --git a/crates/tinyflows-module/src/lib.rs b/crates/tinyflows-module/src/lib.rs index 17c77265..4e6b828e 100644 --- a/crates/tinyflows-module/src/lib.rs +++ b/crates/tinyflows-module/src/lib.rs @@ -12,6 +12,8 @@ mod catalog_authoring; mod catalog_documents; mod catalog_extended_ops; mod catalog_legacy_import; +mod catalog_legacy_batch; +mod catalog_legacy_checkpoint; mod catalog_migrations; mod catalog_service; mod catalog_service_summary; diff --git a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs index 3ffd106e..d1e57776 100644 --- a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs +++ b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs @@ -33,7 +33,7 @@ fn graph_with_dedup() -> WorkflowGraph { } async fn catalog() -> FlowCatalogDocuments { - let backend = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); catalog.ensure().await.unwrap(); catalog diff --git a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs index d344a3fa..51ce25cd 100644 --- a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs +++ b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs @@ -23,7 +23,7 @@ pub(super) struct TestSetup { } pub(super) async fn setup(mode: FaultMode, status: &str) -> TestSetup { - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = Arc::new(FaultingDocuments { inner: scoped.documents().clone(), mode, diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index 511f8d86..e6d708c1 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -249,7 +249,7 @@ struct RunningFixture { } async fn running_fixture() -> RunningFixture { - let backend = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = Arc::clone(backend.documents()); let catalog = FlowCatalogDocuments::new(Arc::clone(&docs)); let graph_json = json!({ diff --git a/crates/tinyflows-module/src/run_manager_tests.rs b/crates/tinyflows-module/src/run_manager_tests.rs index 91521e23..db4561c6 100644 --- a/crates/tinyflows-module/src/run_manager_tests.rs +++ b/crates/tinyflows-module/src/run_manager_tests.rs @@ -15,7 +15,7 @@ fn storage(owner: &str) -> tinyflows_bus::catalog::CatalogStorage { #[tokio::test] async fn terminal_event_seals_cloned_progress_sinks() { - let backend = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); let run_id = format!("sealed-{}", uuid::Uuid::new_v4()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ @@ -96,7 +96,7 @@ fn duplicate_registration_keeps_the_original_cancellation_owner() { #[tokio::test] async fn admission_waiter_rechecks_after_rollback_and_after_success() { - let backend = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); let run_id = format!("admission-{}", uuid::Uuid::new_v4()); let storage = storage("alice"); diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index be80270d..b9190274 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -37,7 +37,7 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( let connection = Connection::connect(bus.connect().await.unwrap()) .await .unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let storage = scoped.documents().clone(); storage .ensure_collection(&CollectionSpec::new("cron.jobs")) diff --git a/crates/tinyflows-module/src/schedule_members_tests.rs b/crates/tinyflows-module/src/schedule_members_tests.rs index 740f6b54..34890634 100644 --- a/crates/tinyflows-module/src/schedule_members_tests.rs +++ b/crates/tinyflows-module/src/schedule_members_tests.rs @@ -206,7 +206,7 @@ async fn store_admission_cap_and_import_operations_are_served() { docs: { use tinyflows_storage::StorageBackend; tinystoragedrivers_testkit::MemoryBackend::new() - .for_scope(&tinyflows_storage::Scope::local()) + .for_scope(&tinyflows_storage::Scope::new("local").expect("valid scope")) .unwrap() .documents() .clone() diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index 631ed697..a7193e2d 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -383,7 +383,7 @@ async fn module_job_storage_preserves_policy_updates_and_run_history() { docs: { use tinyflows_storage::StorageBackend; tinystoragedrivers_testkit::MemoryBackend::new() - .for_scope(&tinyflows_storage::Scope::local()) + .for_scope(&tinyflows_storage::Scope::new("local").expect("valid scope")) .unwrap() .documents() .clone() @@ -428,7 +428,7 @@ mod queue_tests; async fn module_document_store_retries_policy_migration_conflicts_without_losing_seeded_jobs() { use tinyflows_storage::{ Precondition, Scope, StorageBackend}; let (_server, host, _) = connections().await; - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = scoped.documents().clone(); host.serve_at( "/test/Documents".try_into().unwrap(), diff --git a/crates/tinyflows-module/tests/module_artifact_tests.rs b/crates/tinyflows-module/tests/module_artifact_tests.rs index d266e982..fd3a8b6e 100644 --- a/crates/tinyflows-module/tests/module_artifact_tests.rs +++ b/crates/tinyflows-module/tests/module_artifact_tests.rs @@ -360,7 +360,7 @@ async fn compiled_module_serves_validation_without_linking_engine_into_client() // Persist through the actual cdylib. The host forwards document I/O only; // schema, policy and schedule behavior is module-owned. use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = scoped.documents().clone(); let conflict_once = Arc::new(std::sync::atomic::AtomicBool::new(true)); let atomic_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); diff --git a/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs b/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs index 85d52c68..bd537dd5 100644 --- a/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs @@ -39,7 +39,7 @@ async fn compiled_module_serves_manual_worker_lifecycle_through_schedule_abi() { .await .unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let documents = CronDocuments::new(scoped.documents().clone()); let job = documents .add_shell_job( diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 2db83233..9c170b75 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -114,7 +114,7 @@ async fn published_module_migrates_legacy_schedule_records_through_owned_bus_cal broker.attach(Arc::new(transport)); let connection = Connection::connect(Box::new(client)).await.unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = scoped.documents().clone(); docs.ensure_collection(&tinystoragedrivers_core::CollectionSpec::new("cron_jobs")) .await @@ -199,7 +199,7 @@ async fn published_module_migrates_legacy_catalog_records_through_owned_bus_call let (client, transport) = MemoryTransport::pair(); broker.attach(Arc::new(transport)); let connection = Connection::connect(Box::new(client)).await.unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::local()).unwrap(); + let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); let docs = scoped.documents().clone(); for collection in [ "flows_definitions", diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index 5c649710..62449f46 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -9,8 +9,8 @@ publish = false description = "TinyFlows-owned generic document persistence helpers" [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } serde = { workspace = true } serde_json = { workspace = true } async-trait = { workspace = true } @@ -20,4 +20,4 @@ ulid = "1" data-encoding = "2" [dev-dependencies] -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", tag = "v2.0.0-alpha.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } diff --git a/crates/tinyflows-storage/src/blocking.rs b/crates/tinyflows-storage/src/blocking.rs index 58eb46fe..62e56437 100644 --- a/crates/tinyflows-storage/src/blocking.rs +++ b/crates/tinyflows-storage/src/blocking.rs @@ -34,16 +34,18 @@ thread_local! { /// stops the thread once in-flight work finishes. /// /// ``` -/// use tinystoragedrivers_core::{Blocking, MemoryStorage, Precondition, Scope, StorageBackend}; +/// use tinyflows_storage::{Blocking, Precondition, Scope, StorageBackend, StorageError}; +/// use tinystoragedrivers_testkit::MemoryBackend; /// use serde_json::json; /// /// let bridge = Blocking::new()?; -/// let docs = MemoryStorage::new().for_scope(&Scope::local())?.documents().clone(); +/// let storage = MemoryBackend::new().for_scope(&Scope::new("local")?)?; +/// let docs = storage.documents(); /// let version = bridge.run(async move { /// docs.put("settings", "theme", json!({"dark": true}), Precondition::None).await /// })??; /// assert_eq!(version.0, 1); -/// # Ok::<(), tinystoragedrivers_core::StorageError>(()) +/// # Ok::<(), StorageError>(()) /// ``` #[derive(Clone)] pub struct Blocking { @@ -155,5 +157,5 @@ fn startup_error(what: &'static str) -> impl FnOnce(std::io::Error) -> StorageEr } #[cfg(test)] -#[path = "mod_tests.rs"] +#[path = "blocking_tests.rs"] mod tests; diff --git a/crates/tinyflows-storage/src/blocking_tests.rs b/crates/tinyflows-storage/src/blocking_tests.rs new file mode 100644 index 00000000..d0467b19 --- /dev/null +++ b/crates/tinyflows-storage/src/blocking_tests.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn bridge_runs_futures_from_sync_and_current_thread_runtime_contexts() { + let bridge = Blocking::new().expect("bridge starts"); + assert_eq!(bridge.run(async { 42 }).expect("sync call completes"), 42); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + assert_eq!( + runtime + .block_on(async { bridge.run(async { "ready" }) }) + .expect("bridge call from runtime"), + "ready" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn bridge_runs_from_a_multithread_worker() { + let bridge = Blocking::new().expect("bridge starts"); + assert_eq!( + bridge.run(async { 7usize }).expect("bridge call completes"), + 7 + ); +} + +#[test] +fn a_future_cannot_reenter_its_own_bridge() { + let bridge = Blocking::new().expect("bridge starts"); + let inner = bridge.clone(); + let error = bridge + .run(async move { inner.run(async { () }) }) + .expect("outer bridge call completes") + .expect_err("reentry would deadlock"); + assert_eq!(error.kind(), tinystoragedrivers_types::ErrorKind::Backend); +} + +#[test] +fn panicking_jobs_are_reported_to_the_waiting_caller() { + let bridge = Blocking::new().expect("bridge starts"); + let error = bridge + .run(async { panic!("intentional bridge job failure") }) + .expect_err("panicking job closes its reply channel"); + assert_eq!(error.kind(), tinystoragedrivers_types::ErrorKind::Backend); +} diff --git a/crates/tinyflows-storage/src/key.rs b/crates/tinyflows-storage/src/key.rs index 1a41a190..929a7bf8 100644 --- a/crates/tinyflows-storage/src/key.rs +++ b/crates/tinyflows-storage/src/key.rs @@ -32,7 +32,7 @@ //! escaped text hash alike and none can alias another. //! //! ``` -//! use tinystoragedrivers_core::Key; +//! use tinyflows_storage::{Key, StorageError}; //! //! let key = Key::of(["job", "a/b"])?; //! assert_eq!(key.as_str(), "job/a%2Fb"); @@ -41,7 +41,7 @@ //! let long = Key::of(["x".repeat(600)])?; //! assert!(long.is_hashed()); //! assert_eq!(long.split(), None); -//! # Ok::<(), tinystoragedrivers_core::StorageError>(()) +//! # Ok::<(), StorageError>(()) //! ``` use std::fmt; @@ -213,5 +213,5 @@ impl<'de> Deserialize<'de> for Key { } #[cfg(test)] -#[path = "mod_tests.rs"] +#[path = "key_tests.rs"] mod tests; diff --git a/crates/tinyflows-storage/src/key_tests.rs b/crates/tinyflows-storage/src/key_tests.rs new file mode 100644 index 00000000..de3f30fa --- /dev/null +++ b/crates/tinyflows-storage/src/key_tests.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn composite_keys_escape_round_trip_and_hash_long_parts() { + let key = + Key::of(["flow/name", "%state", "~leading", "nul\0byte"]).expect("valid composite key"); + assert_eq!( + key.split().expect("reversible key"), + ["flow/name", "%state", "~leading", "nul\0byte"] + ); + + let long = Key::of(["x".repeat(600)]).expect("long components hash"); + assert!(long.is_hashed()); + assert_eq!(long.as_str().len(), 53); + assert_eq!(long.split(), None); +} + +#[test] +fn serialized_keys_revalidate_document_ids() { + assert!(serde_json::from_str::("\"valid/id\"").is_ok()); + assert!(serde_json::from_str::("\"bad\\u0000id\"").is_err()); +} diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 201d86cf..1ec1bde6 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -57,3 +57,7 @@ pub mod primitives { Ok(()) } } + +#[cfg(test)] +#[path = "schema_tests.rs"] +mod tests; diff --git a/crates/tinyflows-storage/src/primitives/counters_tests.rs b/crates/tinyflows-storage/src/primitives/counters_tests.rs new file mode 100644 index 00000000..92bde806 --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/counters_tests.rs @@ -0,0 +1,36 @@ +use super::*; + +use crate::StorageBackend; +use tinystoragedrivers_testkit::MemoryBackend; +use tinystoragedrivers_types::Scope; + +#[tokio::test] +async fn counters_advance_monotonically_and_raise_imported_values() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("scope")) + .expect("backend"); + let docs = storage.documents(); + docs.ensure_collection(&tinystoragedrivers_types::CollectionSpec::new( + "tsd.counters", + )) + .await + .expect("collection"); + let counters = Counters::new(docs); + assert_eq!(counters.peek("flows", "rows").await.expect("peek"), 0); + assert_eq!(counters.next("flows", "rows").await.expect("first"), 1); + assert_eq!( + counters + .set_at_least("flows", "rows", 8) + .await + .expect("import"), + 8 + ); + assert_eq!( + counters + .set_at_least("flows", "rows", 4) + .await + .expect("do not lower"), + 8 + ); + assert_eq!(counters.next("flows", "rows").await.expect("next"), 9); +} diff --git a/crates/tinyflows-storage/src/primitives/kv_tests.rs b/crates/tinyflows-storage/src/primitives/kv_tests.rs new file mode 100644 index 00000000..ff35122f --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/kv_tests.rs @@ -0,0 +1,55 @@ +use super::*; + +use crate::StorageBackend; +use tinystoragedrivers_testkit::MemoryBackend; +use tinystoragedrivers_types::Scope; + +#[tokio::test] +async fn key_values_are_isolated_by_namespace_and_list_in_key_order() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("scope")) + .expect("backend"); + let docs = storage.documents(); + docs.ensure_collection(&tinystoragedrivers_types::CollectionSpec::new(KV).index( + tinystoragedrivers_types::IndexSpec::new("kv_by_key", ["owner", "ns", "key"]), + )) + .await + .expect("collection"); + let kv = Kv::ns(docs, "flows", "catalog").expect("namespace"); + let other = Kv::ns(docs, "flows", "state").expect("other namespace"); + kv.put("b", &serde_json::json!({"value": 2})) + .await + .expect("write b"); + kv.put("a", &serde_json::json!({"value": 1})) + .await + .expect("write a"); + other + .put("a", &serde_json::json!({"value": 3})) + .await + .expect("write isolated"); + let listed = kv.list::("").await.expect("list"); + assert_eq!( + listed + .iter() + .map(|(key, _)| key.as_str()) + .collect::>(), + ["a", "b"] + ); + assert_eq!( + kv.get::("a") + .await + .expect("read") + .expect("present")["value"], + 1 + ); + assert!(kv.delete("a").await.expect("delete")); + assert!(!kv.delete("a").await.expect("delete missing")); + assert_eq!( + other + .get::("a") + .await + .expect("read other") + .expect("present")["value"], + 3 + ); +} diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 3d5b33da..5b601378 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -7,7 +7,7 @@ //! once it ran out. It deliberately does not. A takeover must read the expired //! record to learn its `epoch` and write `epoch + 1`; if the record vanished, //! the next holder would restart at epoch 1 and a stale holder's -//! [`Fence`] over epoch 1 would hold again. So the record stays, its +//! a fence token over epoch 1 would hold again. So the record stays, its //! `expires_ms` field says when the lease lapses, and `epoch` only ever moves //! forward. @@ -475,7 +475,7 @@ impl Leases { /// the old record and writes `epoch + 1`, so a stale holder's fence at the /// old epoch stays dead. The delete is conditional on holder and epoch, so /// it can only remove the very tenure the caller knows: any takeover bumps - /// the epoch, and a record that moved on is not touched. A [`Fence`] on a + /// the epoch, and a record that moved on is not touched. A holder's fence token on a /// removed record refuses every write (a missing fence document is a /// failed check), so the tenure that was dropped is fenced out for good. /// diff --git a/crates/tinyflows-storage/src/primitives/leases_tests.rs b/crates/tinyflows-storage/src/primitives/leases_tests.rs new file mode 100644 index 00000000..db23963b --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/leases_tests.rs @@ -0,0 +1,42 @@ +use super::*; + +use crate::StorageBackend; +use tinystoragedrivers_testkit::MemoryBackend; +use tinystoragedrivers_types::{CollectionSpec, Scope}; + +#[tokio::test] +async fn lease_epochs_fence_stale_holders_after_expiry_and_release() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("scope")) + .expect("backend"); + let docs = storage.documents(); + docs.ensure_collection(&CollectionSpec::new(LEASES)) + .await + .expect("collection"); + let leases = Leases::new(docs).with_clock(Arc::new(|| 100)); + let first = leases + .acquire("flows", "run", "worker-a", 10) + .await + .expect("acquire") + .expect("first holder wins"); + let epoch = first.epoch; + assert!( + leases + .renew("flows", "run", "worker-b", epoch, 10) + .await + .expect("wrong holder cannot renew") + .is_none() + ); + assert!( + leases + .release("flows", "run", "worker-a", epoch) + .await + .expect("release") + ); + let second = leases + .acquire("flows", "run", "worker-b", 10) + .await + .expect("reacquire") + .expect("released key is available"); + assert!(second.epoch > epoch); +} diff --git a/crates/tinyflows-storage/src/primitives/repo.rs b/crates/tinyflows-storage/src/primitives/repo.rs index 90ba83a3..8011d2c7 100644 --- a/crates/tinyflows-storage/src/primitives/repo.rs +++ b/crates/tinyflows-storage/src/primitives/repo.rs @@ -82,13 +82,13 @@ impl Default for RetryPolicy { /// ``` /// # tokio::runtime::Builder::new_current_thread().build().unwrap().block_on(async { /// use serde::{Deserialize, Serialize}; -/// use tinystoragedrivers_core::primitives::Repo; -/// use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +/// use tinyflows_storage::{DocumentStore, Repo, Scope, StorageBackend, StorageError}; +/// use tinystoragedrivers_testkit::MemoryBackend; /// /// #[derive(Serialize, Deserialize, PartialEq, Debug)] /// struct Job { runs: u32 } /// -/// let storage = MemoryStorage::new().for_scope(&Scope::local())?; +/// let storage = MemoryBackend::new().for_scope(&Scope::new("local")?)?; /// let jobs = Repo::::new(storage.documents(), "cron.jobs")?; /// jobs.update("nightly", |job| Ok(Some(Job { runs: job.map_or(1, |j| j.runs + 1) }))).await?; /// let (job, _version) = jobs @@ -96,7 +96,7 @@ impl Default for RetryPolicy { /// .await? /// .unwrap(); /// assert_eq!(job.runs, 2); -/// # Ok::<(), tinystoragedrivers_core::StorageError>(()) +/// # Ok::<(), StorageError>(()) /// # }).unwrap(); /// ``` pub struct Repo { diff --git a/crates/tinyflows-storage/src/primitives/repo_tests.rs b/crates/tinyflows-storage/src/primitives/repo_tests.rs new file mode 100644 index 00000000..a73c5564 --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/repo_tests.rs @@ -0,0 +1,56 @@ +use super::*; + +use crate::StorageBackend; +use serde::{Deserialize, Serialize}; +use tinystoragedrivers_testkit::MemoryBackend; +use tinystoragedrivers_types::{CollectionSpec, Scope}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct Row { + value: u32, +} + +#[tokio::test] +async fn repo_update_is_atomic_and_conditional_delete_preserves_unmatched_rows() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("scope")) + .expect("backend"); + let docs = storage.documents(); + docs.ensure_collection(&CollectionSpec::new("flows.rows")) + .await + .expect("collection"); + let repo = Repo::::new(docs, "flows.rows") + .expect("repo") + .with_retry(RetryPolicy::immediate()); + let (row, _) = repo + .update("one", |old| { + Ok(Some(Row { + value: old.map_or(1, |row| row.value + 1), + })) + }) + .await + .expect("first update") + .expect("write accepted"); + assert_eq!(row.value, 1); + let (row, _) = repo + .update("one", |old| { + Ok(Some(Row { + value: old.map_or(1, |row| row.value + 1), + })) + }) + .await + .expect("second update") + .expect("write accepted"); + assert_eq!(row.value, 2); + assert!( + !repo + .delete_if("one", |row| row.value == 1) + .await + .expect("conditional delete") + ); + assert!( + repo.delete_if("one", |row| row.value == 2) + .await + .expect("matching delete") + ); +} diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index a134a7f1..bd41a825 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -6,13 +6,14 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt}; use tinystoragedrivers_types::{ - CollectionSpec, ErrorKind, Filter, Meta, Precondition, Query, Result, Sort, StorageError, - WriteOptions, SCHEMA_FIELD, + CollectionSpec, ErrorKind, Filter, ID_FIELD, Meta, Precondition, Query, Result, Sort, + StorageError, WriteOptions, }; use crate::Key; const PAGE_SIZE: usize = 100; +const MIGRATIONS_COLLECTION: &str = "tsd.migrations"; /// Which product scope owns a declared collection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -30,7 +31,9 @@ impl Owner { pub fn new(name: &str) -> Result { let valid = (2..=24).contains(&name.len()) && name.as_bytes().first().is_some_and(u8::is_ascii_lowercase) - && name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); if !valid || name == "tsd" { return Err(StorageError::invalid_input("invalid storage owner name")); } @@ -39,7 +42,9 @@ impl Owner { /// Owner name. #[must_use] - pub fn as_str(&self) -> &str { &self.0 } + pub fn as_str(&self) -> &str { + &self.0 + } } /// A source document passed to a migration step. @@ -70,12 +75,20 @@ impl TargetRecord { /// Create a migrated document. #[must_use] pub fn new(collection: impl Into, id: impl Into, body: Value) -> Self { - Self { collection: collection.into(), id: id.into(), body, created_ms: None } + Self { + collection: collection.into(), + id: id.into(), + body, + created_ms: None, + } } /// Preserve the source creation timestamp. #[must_use] - pub fn created_ms(mut self, value: i64) -> Self { self.created_ms = Some(value); self } + pub fn created_ms(mut self, value: i64) -> Self { + self.created_ms = Some(value); + self + } } type StepFn = dyn Fn(Vec) -> Result> + Send + Sync; @@ -106,12 +119,22 @@ impl Step { to: u32, run: impl Fn(Vec) -> Result> + Send + Sync + 'static, ) -> Self { - Self { id, item, from, to, source: None, run: Arc::new(run) } + Self { + id, + item, + from, + to, + source: None, + run: Arc::new(run), + } } /// Read from a legacy collection rather than the destination. #[must_use] - pub fn source(mut self, name: &'static str) -> Self { self.source = Some(name); self } + pub fn source(mut self, name: &'static str) -> Self { + self.source = Some(name); + self + } } /// One owner’s collections and migrations, opened by that owner. @@ -128,7 +151,13 @@ pub struct Schema { impl Schema { /// Start a declaration for `owner`. #[must_use] - pub fn new(owner: Owner) -> Self { Self { owner, items: Vec::new(), steps: Vec::new() } } + pub fn new(owner: Owner) -> Self { + Self { + owner, + items: Vec::new(), + steps: Vec::new(), + } + } /// Declare a collection and schema version. #[must_use] @@ -138,143 +167,365 @@ impl Schema { /// Declare a fully specified collection and schema version. #[must_use] - pub fn collection_with(mut self, version: u32, spec: CollectionSpec, scope: ScopeClass) -> Self { + pub fn collection_with( + mut self, + version: u32, + spec: CollectionSpec, + scope: ScopeClass, + ) -> Self { self.items.push((spec, version, scope)); self } /// Add an ordered collection migration. #[must_use] - pub fn step(mut self, step: Step) -> Self { self.steps.push(step); self } + pub fn step(mut self, step: Step) -> Self { + self.steps.push(step); + self + } /// Pure collection declarations for host bootstrap and documentation. #[must_use] pub fn collections(&self) -> Vec { - let mut specs = self.items.iter().map(|(spec, _, _)| spec.clone()).collect::>(); - specs.push(CollectionSpec::new(self.migrations_collection())); + let mut specs = self + .items + .iter() + .map(|(spec, _, _)| spec.clone()) + .collect::>(); + specs.push(CollectionSpec::new(MIGRATIONS_COLLECTION)); specs } - fn migrations_collection(&self) -> String { format!("{}.migrations", self.owner.as_str()) } - /// Ensure this owner’s collections, then migrate its data in place. /// - /// A separate `.migrations` collection records one progress row per - /// step. Backends are supplied by the caller; this method never constructs - /// an engine or an in-memory fallback. - pub async fn migrate(&self, docs: &Arc, options: &MigrateOptions) -> Result { - if options.batch_size == 0 { return Err(StorageError::invalid_input("migration batch size must be at least 1")); } - for spec in self.collections() { docs.ensure_collection(&spec).await?; } - let migrations = Migrations { docs: Arc::clone(docs), collection: self.migrations_collection() }; + /// The shared `tsd.migrations` collection records one progress row per + /// owner, item and step. Backends are supplied by the caller; this method + /// never constructs an engine or an in-memory fallback. + pub async fn migrate( + &self, + docs: &Arc, + options: &MigrateOptions, + ) -> Result { + if options.batch_size == 0 { + return Err(StorageError::invalid_input( + "migration batch size must be at least 1", + )); + } + for spec in self.collections() { + docs.ensure_collection(&spec).await?; + } + let migrations = Migrations { + docs: Arc::clone(docs), + }; let mut report = MigrationReport::default(); for step in &self.steps { - if migrations.done(self.owner.as_str(), step.id).await? { report.already_done += 1; continue; } - let progress_id = Key::of([self.owner.as_str(), step.id])?.as_str().to_owned(); - let progress = migrations.docs.get_as::(&migrations.collection, &progress_id).await?; + let progress_key = Key::of([self.owner.as_str(), step.item, step.id])? + .as_str() + .to_owned(); + let progress = migrations.load(&progress_key).await?; + if progress.as_ref().is_some_and(|p| p.done) { + report.already_done += 1; + continue; + } + let source = step.source.unwrap_or(step.item); + if source != step.item { + docs.ensure_collection(&CollectionSpec::new(source)).await?; + } let mut query = Query::filter(Filter::default()) - .sort(Sort::asc("_id")) + .sort(Sort::asc(ID_FIELD)) .limit(options.batch_size); - if let Some(cursor) = progress.and_then(|p| p.doc.cursor) { + if let Some(cursor) = progress.as_ref().and_then(|p| p.cursor.clone()) { query = query.after(tinystoragedrivers_types::Cursor(cursor)); } - let source = step.source.unwrap_or(step.item); - let mut written = 0_u64; + let mut written = progress.as_ref().map_or(0, |saved| saved.written); loop { let page = docs.query(source, &query).await?; - if page.items.is_empty() { break; } + if page.items.is_empty() { + break; + } let last_cursor = page.next.clone(); - let inputs = page.items.iter() - .filter(|row| row.meta.schema < step.to) - .map(|row| SourceRecord { id: row.id.clone(), body: row.doc.clone(), meta: row.meta.clone() }) + let inputs = page + .items + .iter() + .filter(|row| source != step.item || row.meta.schema < step.to) + .map(|row| SourceRecord { + id: row.id.clone(), + body: row.doc.clone(), + meta: row.meta.clone(), + }) .collect(); let outputs = (step.run)(inputs)?; - let output_ids: std::collections::HashSet<(String, String)> = outputs.iter().map(|r| (r.collection.clone(), r.id.clone())).collect(); - for mut target in outputs { - if !target.body.is_object() { return Err(StorageError::invalid_input("migration output must be an object")); } - target.body[SCHEMA_FIELD] = step.to.into(); - let options = target.created_ms.map_or_else(WriteOptions::new, |ms| WriteOptions::new().created_ms(ms)); - let source_row = (source == step.item) + let output_ids: std::collections::HashSet<(String, String)> = outputs + .iter() + .map(|r| (r.collection.clone(), r.id.clone())) + .collect(); + for target in outputs { + if !target.body.is_object() { + return Err(StorageError::invalid_input( + "migration output must be an object", + )); + } + let output_schema = self.target_schema(step, &target)?; + let source_row = (source == step.item && target.collection == step.item) .then(|| page.items.iter().find(|row| row.id == target.id)); - let precondition = source_row.flatten() - .map_or(Precondition::Absent, |row| Precondition::Version(row.version)); - match docs.put_with(&target.collection, &target.id, target.body.clone(), precondition, options).await { + let source_row = source_row.flatten(); + let precondition = source_row.map_or(Precondition::Absent, |row| { + Precondition::Version(row.version) + }); + let created_ms = target + .created_ms + .or_else(|| source_row.map(|row| row.meta.created_ms)); + let mut options = WriteOptions::new(); + options = options.schema(output_schema); + if let Some(created_ms) = created_ms { + options = options.created_ms(created_ms); + } + if let Some(expires_ms) = source_row.and_then(|row| row.meta.expires_ms) { + options = options.expires_ms(expires_ms); + } + match docs + .put_with( + &target.collection, + &target.id, + target.body.clone(), + precondition, + options, + ) + .await + { Ok(_) => written += 1, - Err(error) if matches!(error.kind(), ErrorKind::Conflict | ErrorKind::AlreadyExists) => { + Err(error) + if matches!( + error.kind(), + ErrorKind::Conflict | ErrorKind::AlreadyExists + ) => + { let existing = docs.get(&target.collection, &target.id).await?; - if existing.is_none_or(|row| row.doc != target.body) { return Err(error); } + let expected_expiry = + source_row.and_then(|source| source.meta.expires_ms); + let identical = existing.is_some_and(|row| { + row.doc == target.body + && row.meta.schema == output_schema + && created_ms + .is_none_or(|created| row.meta.created_ms == created) + && row.meta.expires_ms == expected_expiry + }); + if !identical { + return Err(error); + } written += 1; } Err(error) => return Err(error), } } for row in &page.items { - if row.meta.schema >= step.to { continue; } - if source != step.item || !output_ids.contains(&(step.item.to_owned(), row.id.clone())) { - docs.delete(source, &row.id, Precondition::Version(row.version)).await?; + if (source != step.item || row.meta.schema < step.to) + && (source != step.item + || !output_ids.contains(&(step.item.to_owned(), row.id.clone()))) + { + match docs + .delete(source, &row.id, Precondition::Version(row.version)) + .await + { + Err(error) + if error.kind() == ErrorKind::Conflict + && docs.get(source, &row.id).await?.is_none() => {} + other => { + other?; + } + } } } let cursor = last_cursor.map(|cursor| cursor.0); - migrations.save(&progress_id, &Progress { owner: self.owner.as_str().to_owned(), step: step.id.to_owned(), cursor, done: false }).await?; - if let Some(next) = page.next { query = query.after(next); } else { break; } + migrations + .save( + &progress_key, + &Progress { + owner: self.owner.as_str().to_owned(), + step: step.id.to_owned(), + from: step.from, + to: step.to, + cursor, + written, + done: false, + }, + ) + .await?; + if let Some(next) = page.next { + query = query.after(next); + } else { + break; + } } - migrations.save(&progress_id, &Progress { owner: self.owner.as_str().to_owned(), step: step.id.to_owned(), cursor: None, done: true }).await?; - report.ran.push(StepReport { owner: self.owner.as_str().to_owned(), item: step.item.to_owned(), step: step.id.to_owned(), written }); + migrations + .save( + &progress_key, + &Progress { + owner: self.owner.as_str().to_owned(), + step: step.id.to_owned(), + from: step.from, + to: step.to, + cursor: None, + written, + done: true, + }, + ) + .await?; + report.ran.push(StepReport { + owner: self.owner.as_str().to_owned(), + item: step.item.to_owned(), + step: step.id.to_owned(), + written, + }); } Ok(report) } + + fn target_schema(&self, step: &Step, target: &TargetRecord) -> Result { + let owned = target + .collection + .strip_prefix(self.owner.as_str()) + .is_some_and(|rest| rest.starts_with('.')); + let version = self + .items + .iter() + .find(|(spec, _, _)| spec.name == target.collection) + .map(|(_, version, _)| *version) + .filter(|_| owned) + .ok_or_else(|| { + StorageError::invalid_input( + "migration output targets an undeclared owner collection", + ) + })?; + Ok(if target.collection == step.item { + step.to + } else { + version + }) + } } /// Batch size for owner migration runs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct MigrateOptions { batch_size: usize } +pub struct MigrateOptions { + batch_size: usize, +} impl MigrateOptions { /// Defaults to 100 documents per batch. #[must_use] - pub const fn new() -> Self { Self { batch_size: PAGE_SIZE } } + pub const fn new() -> Self { + Self { + batch_size: PAGE_SIZE, + } + } /// Set the number of source documents read per batch. #[must_use] - pub const fn batch_size(mut self, batch_size: usize) -> Self { self.batch_size = batch_size; self } + pub const fn batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = batch_size; + self + } +} +impl Default for MigrateOptions { + fn default() -> Self { + Self::new() + } } -impl Default for MigrateOptions { fn default() -> Self { Self::new() } } /// Summary of an owner migration run. #[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct MigrationReport { /// Steps completed by this invocation. - pub ran: Vec, /// Steps already marked complete. - pub already_done: usize } +pub struct MigrationReport { + /// Steps completed by this invocation. + pub ran: Vec, + /// Steps already marked complete. + pub already_done: usize, +} /// A completed migration step. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StepReport { /// Owner name. - pub owner: String, /// Collection being migrated. - pub item: String, /// Stable step identifier. - pub step: String, /// Number of destination records written. - pub written: u64 } +pub struct StepReport { + /// Owner name. + pub owner: String, + /// Collection being migrated. + pub item: String, + /// Stable step identifier. + pub step: String, + /// Number of destination records written. + pub written: u64, +} #[derive(Debug, Clone, Serialize, Deserialize)] -struct Progress { owner: String, step: String, cursor: Option, done: bool } +struct Progress { + owner: String, + step: String, + from: u32, + to: u32, + cursor: Option, + written: u64, + done: bool, +} /// Per-owner migration completion markers. #[derive(Debug, Clone)] -pub struct Migrations { docs: Arc, collection: String } +pub struct Migrations { + docs: Arc, +} impl Migrations { - /// Bind markers to the caller-provided document store and private collection. + /// Bind markers to the caller-provided document store and owner. #[must_use] - pub fn new(docs: &Arc, owner: &str) -> Self { Self { docs: Arc::clone(docs), collection: format!("{owner}.migrations") } } + pub fn new(docs: &Arc, _owner: &str) -> Self { + Self { + docs: Arc::clone(docs), + } + } /// Whether a step has completed. pub async fn done(&self, owner: &str, step: &str) -> Result { + self.ensure().await?; let id = Key::of([owner, step])?.as_str().to_owned(); - Ok(self.docs.get_as::(&self.collection, &id).await?.is_some_and(|row| row.doc.done)) + Ok(self + .docs + .get_as::(MIGRATIONS_COLLECTION, &id) + .await? + .is_some_and(|row| row.doc.done)) } /// Mark an external import step complete. pub async fn mark_done(&self, owner: &str, step: &str) -> Result<()> { + self.ensure().await?; let id = Key::of([owner, step])?.as_str().to_owned(); - self.save(&id, &Progress { owner: owner.to_owned(), step: step.to_owned(), cursor: None, done: true }).await + self.save( + &id, + &Progress { + owner: owner.to_owned(), + step: step.to_owned(), + from: 0, + to: 0, + cursor: None, + written: 0, + done: true, + }, + ) + .await + } + + async fn load(&self, id: &str) -> Result> { + Ok(self + .docs + .get_as::(MIGRATIONS_COLLECTION, id) + .await? + .map(|row| row.doc)) + } + + async fn ensure(&self) -> Result<()> { + self.docs + .ensure_collection(&CollectionSpec::new(MIGRATIONS_COLLECTION)) + .await } async fn save(&self, id: &str, progress: &Progress) -> Result<()> { - self.docs.put_as(&self.collection, id, progress, Precondition::None).await.map(|_| ()) + self.docs + .put_as(MIGRATIONS_COLLECTION, id, progress, Precondition::None) + .await + .map(|_| ()) } } diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs new file mode 100644 index 00000000..fd402001 --- /dev/null +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -0,0 +1,164 @@ +use super::*; + +use serde_json::json; +use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend; + +#[tokio::test] +async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + docs.ensure_collection(&CollectionSpec::new("flows.items")) + .await + .expect("source collection"); + docs.put_with( + "flows.items", + "one", + json!({"value": "before"}), + Precondition::Absent, + WriteOptions::new() + .schema(1) + .created_ms(123) + .expires_ms(999), + ) + .await + .expect("seed source"); + + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .step(Step::new("items_v2", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| TargetRecord::new("flows.items", row.id, json!({"value": "after"}))) + .collect()) + })); + + let first = schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("migration succeeds"); + let stored = docs.get("flows.items", "one").await.expect("read migrated row"); + assert!(stored.is_some(), "migration report: {first:?}"); + let stored = stored.expect("row retained"); + assert_eq!(stored.doc["value"], "after"); + assert_eq!(stored.meta.schema, 2); + assert_eq!(stored.meta.created_ms, 123); + assert_eq!(stored.meta.expires_ms, Some(999)); + assert_eq!(first.ran.len(), 1); + + let second = schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("completed migration is resumable"); + assert_eq!(second.already_done, 1); + assert_eq!( + docs.get("flows.items", "one") + .await + .expect("read after replay") + .expect("row retained") + .doc["value"], + "after" + ); +} + +#[tokio::test] +async fn owner_migration_rejects_same_id_conflicts_in_a_different_declared_collection() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + docs.ensure_collection(&CollectionSpec::new("legacy.items")) + .await + .expect("legacy source collection"); + docs.put_with( + "legacy.items", + "same-id", + json!({"value": "legacy"}), + Precondition::Absent, + WriteOptions::new().schema(1), + ) + .await + .expect("seed source"); + docs.put_with( + "flows.other", + "same-id", + json!({"value": "stale target"}), + Precondition::Absent, + WriteOptions::new().schema(1), + ) + .await + .expect("seed target with matching source version"); + + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .collection("flows.other", 9, ScopeClass::Tenant) + .step( + Step::new("items_v2", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| { + TargetRecord::new("flows.other", row.id, json!({"value": "migrated"})) + }) + .collect()) + }) + .source("legacy.items"), + ); + + let error = schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect_err("a different collection's same-id row must not be overwritten"); + assert_eq!(error.kind(), ErrorKind::Conflict); + assert_eq!( + docs.get("flows.other", "same-id") + .await + .expect("read target") + .expect("target remains") + .doc["value"], + "stale target" + ); + assert!( + docs.get("legacy.items", "same-id") + .await + .expect("read source") + .is_some() + ); +} + +#[tokio::test] +async fn owner_migration_reads_compatible_shared_progress_records() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + let key = Key::of(["flows", "flows.items", "items_v2"]).expect("progress key"); + docs.put_as( + "tsd.migrations", + key.as_str(), + &json!({ + "owner": "flows", + "step": "items_v2", + "from": 1, + "to": 2, + "cursor": null, + "written": 0, + "done": true + }), + Precondition::Absent, + ) + .await + .expect("seed legacy progress record"); + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .step(Step::new("items_v2", "flows.items", 1, 2, |_| { + panic!("completed progress must skip the migration") + })); + + let report = schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("legacy progress remains readable"); + assert_eq!(report.already_done, 1); +} diff --git a/crates/tinyflows/Cargo.toml b/crates/tinyflows/Cargo.toml index 168845df..220e9acf 100644 --- a/crates/tinyflows/Cargo.toml +++ b/crates/tinyflows/Cargo.toml @@ -2,7 +2,7 @@ name = "tinyflows" version = "0.8.28" edition = "2024" -rust-version = "1.85" +rust-version = "1.88" license = "GPL-3.0-or-later" description = "A Rust-based workflow management solution." readme = "README.md" diff --git a/scripts/check-storage-engine-free.sh b/scripts/check-storage-engine-free.sh new file mode 100755 index 00000000..8375bd46 --- /dev/null +++ b/scripts/check-storage-engine-free.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +engine_pattern='^(rusqlite|libsqlite3-sys|mongodb|bson|aws-sdk-s3|lapin|amq-protocol|sqlx|redb|sled|rocksdb|tinystoragedrivers-(document|kv|blob|queue|secrets|connect)|tinystoragedrivers)$' +engine_names() { + awk '{print $1}' <<<"$1" | grep -xE "$engine_pattern" || true +} + +# Keep the detector itself covered: it must catch an engine and accept public +# port crates and ordinary path dependencies. +if [[ -z "$(engine_names $'tinystoragedrivers-document v2.0.0\n')" ]]; then + echo "engine detector self-test failed to catch a backend crate" >&2 + exit 1 +fi +if [[ -n "$(engine_names $'tinystoragedrivers-api v2.0.0\ntinyflows-storage v0.1.0\n')" ]]; then + echo "engine detector self-test rejected an API-only tree" >&2 + exit 1 +fi + +packages=$(cargo metadata --no-deps --format-version 1 | python3 -c ' +import json, sys +metadata = json.load(sys.stdin) +library_kinds = {"lib", "rlib", "dylib", "cdylib", "staticlib", "proc-macro"} +for package in metadata["packages"]: + if any(set(target["kind"]) & library_kinds for target in package["targets"]): + print(package["name"]) +') +if [[ -z "$packages" ]]; then + echo "cargo metadata returned no library targets; refusing a vacuous engine check" >&2 + exit 1 +fi + +while IFS= read -r package; do + [[ -n "$package" ]] || continue + tree=$(cargo tree -p "$package" --all-features -e normal,build --prefix none) + engines=$(engine_names "$tree") + if [[ -n "$engines" ]]; then + printf 'storage implementation reached by %s:\n%s\n' "$package" "$engines" >&2 + exit 1 + fi +done <<<"$packages" From c05bf51296f6c16bf5dcd1a1abdb2cccea666721 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 16:50:41 +0300 Subject: [PATCH 21/64] test(schema): add assertions for owner migration test Adds assertions to verify the state of documents before and after the owner migration runs. This ensures that the migration correctly preserves document metadata and handles the expected number of rows during the step execution. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema_tests.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index fd402001..9d3e78ec 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -25,10 +25,17 @@ async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata( ) .await .expect("seed source"); + let before = docs + .query("flows.items", &Query::filter(Filter::default()).limit(100)) + .await + .expect("query before migration"); + assert_eq!(before.items.len(), 1); + assert_eq!(before.items[0].meta.schema, 1); let schema = Schema::new(Owner::new("flows").expect("valid owner")) .collection("flows.items", 2, ScopeClass::Tenant) .step(Step::new("items_v2", "flows.items", 1, 2, |rows| { + assert_eq!(rows.len(), 1, "one source row should be passed to the step"); Ok(rows .into_iter() .map(|row| TargetRecord::new("flows.items", row.id, json!({"value": "after"}))) @@ -39,7 +46,10 @@ async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata( .migrate(&docs, &MigrateOptions::default()) .await .expect("migration succeeds"); - let stored = docs.get("flows.items", "one").await.expect("read migrated row"); + let stored = docs + .get("flows.items", "one") + .await + .expect("read migrated row"); assert!(stored.is_some(), "migration report: {first:?}"); let stored = stored.expect("row retained"); assert_eq!(stored.doc["value"], "after"); From d026f80d7d84a82913a2e9cd68251f88c24f69f1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 17:41:09 +0300 Subject: [PATCH 22/64] chore: files changed CLAUDE.md,Cargo.lock,Cargo.toml,crates/tinyflows-adaptive/src/drivers/ledger_te Auto-committed-on: dragonfly --- CLAUDE.md | 3 +- Cargo.lock | 106 +- Cargo.toml | 1 - .../src/drivers/ledger_tests.rs | 4 +- .../src/drivers/migrate_tests.rs | 31 +- .../src/drivers/vault_tests.rs | 4 +- crates/tinyflows-adaptive/src/storage.rs | 4 +- .../tinyflows-adaptive/src/storage_tests.rs | 4 +- .../tests/driver/driver_part_01_tests.rs | 4 +- .../tests/support/driver_store.rs | 3 +- crates/tinyflows-bus/src/schedule/mod.rs | 2 +- crates/tinyflows-drivers/README.md | 28 +- .../src/catalog/definitions_tests.rs | 4 +- .../src/catalog/drafts_tests.rs | 2 +- .../tinyflows-drivers/src/catalog/import.rs | 3 +- .../tinyflows-drivers/src/catalog/kv_tests.rs | 2 +- .../src/catalog/migrate_tests.rs | 30 +- crates/tinyflows-drivers/src/catalog/mod.rs | 2 +- .../src/catalog/revisions_gap_tests.rs | 4 +- .../src/catalog/runs_lifecycle_tests.rs | 5 +- .../src/catalog/runs_recovery_lease_tests.rs | 4 +- .../src/catalog/runs_replace_tests.rs | 4 +- .../src/catalog/test_support.rs | 11 +- .../src/checkpoint_import_tests.rs | 4 +- .../src/checkpoint_layout_tests.rs | 5 +- .../tinyflows-drivers/src/checkpoint_tests.rs | 5 +- .../src/schedule/admission.rs | 15 +- .../src/schedule/admission_tests.rs | 3 +- .../src/schedule/claim_removal_tests.rs | 3 +- .../src/schedule/claims_compaction_tests.rs | 5 +- .../src/schedule/claims_safety_tests.rs | 3 +- .../src/schedule/claims_tests.rs | 3 +- .../tinyflows-drivers/src/schedule/import.rs | 4 +- .../src/schedule/import_tests.rs | 2 +- .../src/schedule/manual_claim_fence_tests.rs | 30 +- .../src/schedule/manual_claim_tests.rs | 2 +- .../src/schedule/migrate_tests.rs | 32 +- crates/tinyflows-drivers/src/schedule/mod.rs | 5 +- .../src/schedule/mod_tests.rs | 4 +- .../src/schedule/removal_claims_tests.rs | 3 +- .../src/schedule/test_support.rs | 11 +- .../src/state_store_tests.rs | 4 +- .../src/storage/kv_import_tests.rs | 6 +- .../src/store/authoring_tests.rs | 3 +- .../src/store/concurrency_tests.rs | 3 +- .../src/store/history_tests.rs | 3 +- crates/tinyflows-drivers/src/store/import.rs | 3 +- .../src/store/import_tests.rs | 2 +- .../src/store/locks_tests.rs | 5 +- .../src/store/migrate_tests.rs | 35 +- .../tinyflows-drivers/src/store/mod_tests.rs | 5 +- .../src/store/notes_tests.rs | 5 +- .../src/store/proposals_tests.rs | 5 +- .../tinyflows-drivers/src/store/runs_tests.rs | 5 +- .../src/store/test_support.rs | 11 +- crates/tinyflows-import/Cargo.toml | 67 -- crates/tinyflows-import/src/catalog.rs | 46 - crates/tinyflows-import/src/catalog_tests.rs | 328 ------ .../tinyflows-import/src/checkpoint_reader.rs | 385 ------- .../src/checkpoint_reader_tests.rs | 404 ------- crates/tinyflows-import/src/checkpoints.rs | 85 -- .../tinyflows-import/src/checkpoints_tests.rs | 415 ------- crates/tinyflows-import/src/cron.rs | 56 - crates/tinyflows-import/src/cron_tests.rs | 240 ---- crates/tinyflows-import/src/flow_reader.rs | 502 -------- .../tinyflows-import/src/flow_reader_tests.rs | 348 ------ crates/tinyflows-import/src/legacy_batch.rs | 326 ------ .../src/legacy_batch_tests.rs | 125 -- .../tinyflows-import/src/legacy_checkpoint.rs | 209 ---- crates/tinyflows-import/src/legacy_cron.rs | 291 ----- crates/tinyflows-import/src/legacy_db.rs | 105 -- .../tinyflows-import/src/legacy_db_tests.rs | 116 -- crates/tinyflows-import/src/lib.rs | 128 --- crates/tinyflows-import/src/report.rs | 134 --- crates/tinyflows-import/src/report_tests.rs | 40 - crates/tinyflows-import/src/state.rs | 57 - crates/tinyflows-import/src/state_tests.rs | 140 --- crates/tinyflows-import/src/test_support.rs | 185 --- crates/tinyflows-import/src/workflow_store.rs | 283 ----- .../src/workflow_store_tests.rs | 337 ------ .../tests/catalog_feature_contract_tests.rs | 37 - .../tinyflows-import/tests/fixtures/README.md | 21 - .../fixtures/catalog/drafts/draft-1.json | 10 - .../fixtures/catalog/drafts/draft-2.json | 15 - .../tests/fixtures/catalog/flows.db | Bin 69632 -> 0 bytes .../tests/fixtures/checkpoints/checkpoints.db | Bin 49152 -> 0 bytes .../file/%52un%20%54wo%2F%55pper.jsonl | 1 - .../fixtures/checkpoints/file/run-1.jsonl | 4 - .../checkpoints/file/run-1.writes.jsonl | 3 - .../tests/fixtures/cron/jobs.db | Bin 40960 -> 0 bytes .../tests/fixtures/expected.json | 1013 ----------------- ...8346524fadb143726d256b1a045303fc67717.json | 1 - ...eca7c7f0a67801f7693eb2b8ccba347c58f11.json | 1 - ...0002-d32c0dd55d3aa699292153bc11572798.json | 53 - ...0000-e8ec8f20403e6ee93f6db7acc1a07b98.json | 53 - ...0001-cf5377d354cea8a5b39a7e89513802ec.json | 53 - .../workflow-store/state/journal/sweep.json | 31 - ...0000-83da8a7edd825c2bce8cd7392068472f.json | 21 - .../workflow-store/state/runs/run-a.json | 8 - .../workflow-store/state/runs/run-b.json | 8 - .../workflow-store/state/runs/run-c.json | 8 - .../workflow-store/workflows/deploy.json | 46 - .../workflow-store/workflows/sweep.json | 46 - crates/tinyflows-module/Cargo.toml | 1 - crates/tinyflows-module/README.md | 14 +- .../src/caps_tests/scope_tests.rs | 2 +- .../tinyflows-module/src/catalog_documents.rs | 7 +- .../src/catalog_migrations.rs | 11 +- .../src/catalog_service_tests.rs | 9 +- crates/tinyflows-module/src/lib.rs | 2 +- .../src/run_manager/dedup_settlement_tests.rs | 6 +- .../events_persistence_test_support.rs | 21 +- .../src/run_manager_actor_recovery_tests.rs | 6 +- .../tinyflows-module/src/run_manager_tests.rs | 12 +- .../src/schedule_clock_tests.rs | 4 +- .../src/schedule_documents.rs | 7 +- .../src/schedule_documents_tests.rs | 8 +- ...e_manual_worker_lifecycle_service_tests.rs | 4 +- .../schedule_manual_worker_service_tests.rs | 8 +- .../src/schedule_members_tests.rs | 3 +- .../src/schedule_service_tests.rs | 12 +- crates/tinyflows-module/src/schedule_store.rs | 10 +- .../src/schedule_store_tests.rs | 4 +- .../tests/fixtures/storage/README.md | 51 + .../fixtures/storage/cron-job-v0.8.28.json | 31 + .../fixtures/storage/cron-job-v0.8.8.json | 20 + .../storage/generators/generate_v0.8.28.rs | 29 + .../storage/generators/generate_v0.8.8.rs | 27 + .../tests/module_artifact_tests.rs | 7 +- .../tests/run_artifact/document_callback.rs | 4 +- .../tests/run_artifact/support.rs | 3 +- .../catalog_legacy_import_artifact_tests.rs | 3 +- .../tests/support/document_callbacks.rs | 6 +- .../support/manual_worker_artifact_tests.rs | 7 +- .../tests/support/schedule_v4_artifact.rs | 2 +- .../support/schedule_v4_delivery_artifact.rs | 2 +- .../support/schedule_v4_manual_artifact.rs | 2 +- .../support/schedule_v4_recovery_artifact.rs | 2 +- .../support/scheduler_stop_artifact_tests.rs | 3 +- .../storage_migration_artifact_tests.rs | 134 ++- crates/tinyflows-storage/src/blocking.rs | 3 +- crates/tinyflows-storage/src/lib.rs | 17 +- .../src/primitives/counters.rs | 2 +- crates/tinyflows-storage/src/primitives/kv.rs | 2 +- .../src/primitives/leases.rs | 4 +- crates/tinyflows-storage/src/schema_tests.rs | 53 +- docs/storage-crate-split.md | 29 +- docs/tinyflows-bus-design.md | 10 +- scripts/check-legacy-import-modes.sh | 23 - scripts/check-storage-engine-free.sh | 10 +- 150 files changed, 681 insertions(+), 7240 deletions(-) delete mode 100644 crates/tinyflows-import/Cargo.toml delete mode 100644 crates/tinyflows-import/src/catalog.rs delete mode 100644 crates/tinyflows-import/src/catalog_tests.rs delete mode 100644 crates/tinyflows-import/src/checkpoint_reader.rs delete mode 100644 crates/tinyflows-import/src/checkpoint_reader_tests.rs delete mode 100644 crates/tinyflows-import/src/checkpoints.rs delete mode 100644 crates/tinyflows-import/src/checkpoints_tests.rs delete mode 100644 crates/tinyflows-import/src/cron.rs delete mode 100644 crates/tinyflows-import/src/cron_tests.rs delete mode 100644 crates/tinyflows-import/src/flow_reader.rs delete mode 100644 crates/tinyflows-import/src/flow_reader_tests.rs delete mode 100644 crates/tinyflows-import/src/legacy_batch.rs delete mode 100644 crates/tinyflows-import/src/legacy_batch_tests.rs delete mode 100644 crates/tinyflows-import/src/legacy_checkpoint.rs delete mode 100644 crates/tinyflows-import/src/legacy_cron.rs delete mode 100644 crates/tinyflows-import/src/legacy_db.rs delete mode 100644 crates/tinyflows-import/src/legacy_db_tests.rs delete mode 100644 crates/tinyflows-import/src/lib.rs delete mode 100644 crates/tinyflows-import/src/report.rs delete mode 100644 crates/tinyflows-import/src/report_tests.rs delete mode 100644 crates/tinyflows-import/src/state.rs delete mode 100644 crates/tinyflows-import/src/state_tests.rs delete mode 100644 crates/tinyflows-import/src/test_support.rs delete mode 100644 crates/tinyflows-import/src/workflow_store.rs delete mode 100644 crates/tinyflows-import/src/workflow_store_tests.rs delete mode 100644 crates/tinyflows-import/tests/catalog_feature_contract_tests.rs delete mode 100644 crates/tinyflows-import/tests/fixtures/README.md delete mode 100644 crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-1.json delete mode 100644 crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-2.json delete mode 100644 crates/tinyflows-import/tests/fixtures/catalog/flows.db delete mode 100644 crates/tinyflows-import/tests/fixtures/checkpoints/checkpoints.db delete mode 100644 crates/tinyflows-import/tests/fixtures/checkpoints/file/%52un%20%54wo%2F%55pper.jsonl delete mode 100644 crates/tinyflows-import/tests/fixtures/checkpoints/file/run-1.jsonl delete mode 100644 crates/tinyflows-import/tests/fixtures/checkpoints/file/run-1.writes.jsonl delete mode 100644 crates/tinyflows-import/tests/fixtures/cron/jobs.db delete mode 100644 crates/tinyflows-import/tests/fixtures/expected.json delete mode 100644 crates/tinyflows-import/tests/fixtures/file-state/23f934e93be1ce4552c02f0afd23f913129e848ae8b554dc159dae21eb02b10e/46a4eebd20d881ecfc0ecb54f6d8346524fadb143726d256b1a045303fc67717.json delete mode 100644 crates/tinyflows-import/tests/fixtures/file-state/23f934e93be1ce4552c02f0afd23f913129e848ae8b554dc159dae21eb02b10e/704c85b17a4304aa541a0fbb7a5eca7c7f0a67801f7693eb2b8ccba347c58f11.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/definitions/3854983f88763a4631f806cfdedde1b4d5dff5f5194b6fdd75998417f9f4d845/revisions/retired/1791849384244-000000000002-d32c0dd55d3aa699292153bc11572798.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/definitions/3854983f88763a4631f806cfdedde1b4d5dff5f5194b6fdd75998417f9f4d845/revisions/sweep/1791849384244-000000000000-e8ec8f20403e6ee93f6db7acc1a07b98.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/definitions/3854983f88763a4631f806cfdedde1b4d5dff5f5194b6fdd75998417f9f4d845/revisions/sweep/1791849384244-000000000001-cf5377d354cea8a5b39a7e89513802ec.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/journal/sweep.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/proposals/0000000000005-000000000000-83da8a7edd825c2bce8cd7392068472f.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/runs/run-a.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/runs/run-b.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/state/runs/run-c.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/workflows/deploy.json delete mode 100644 crates/tinyflows-import/tests/fixtures/workflow-store/workflows/sweep.json create mode 100644 crates/tinyflows-module/tests/fixtures/storage/README.md create mode 100644 crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.28.json create mode 100644 crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.8.json create mode 100644 crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.28.rs create mode 100644 crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.8.rs delete mode 100755 scripts/check-legacy-import-modes.sh diff --git a/CLAUDE.md b/CLAUDE.md index 78f5ad4e..2e678928 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,8 +23,7 @@ uses, so a reader who knows one knows them all. | --- | --- | | `crates/tinyflows` | The engine. One graph: validate → compile → run. Published to crates.io. | | `crates/tinyflows-catalog` | The saved-flow model *around* a graph — flows, revisions, runs, drafts, suggestions — the in-process run/build cancellation registries, the format importers (n8n), and the save/run safety predicates. Storage-free. | -| `crates/tinyflows-drivers` | The catalog (flows, drafts, runs, state), the cron job/run store, the engine's checkpointer and state store, and `WorkflowStore` (workflows, revisions, runs, notes, proposals) on `tinystoragedrivers` document ports. Unpublished: it names the storage crates by git tag, which a published crate cannot carry. Depends on `tinystoragedrivers-core` only; the host picks the backend. | -| `crates/tinyflows-import` | One-time, read-only import of the retired stores (`flows.db`, `jobs.db`, `checkpoints.db`, the draft/`File*` JSON and JSONL stores) into the `tinyflows-drivers` stores, id for id and only where absent. Unpublished; reads SQLite through the SQLite driver's native handle. | +| `crates/tinyflows-drivers` | The catalog (flows, drafts, runs, state), the cron job/run store, the engine's checkpointer and state store, and `WorkflowStore` (workflows, revisions, runs, notes, proposals) on `tinystoragedrivers` document ports. It uses public storage API/types through the private `tinyflows-storage` helper; the host picks the backend. | | `crates/tinyflows-copilot` | The authoring copilot's *words*: the `workflow_builder` / `flow_discovery` standing archetypes, and the turn brief that opens a builder turn. Names no tool trait, no agent registry, no model client. | | `crates/tinyflows-adaptive` | The adaptive loop over the engine: select or author a workflow, run it, judge it, learn. | | `crates/tinyflows-schedule` | Schedule model, pure next-run logic (cron / interval / one-shot, time zones, active hours, cadence floor), and the schedule runtime. Persistence and configuration are supplied by the host through `tinyflows-drivers` and `tinyflows-module`. | diff --git a/Cargo.lock b/Cargo.lock index dd462ecc..3c95f392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,27 +2016,6 @@ dependencies = [ "syn 3.0.9", ] -[[package]] -name = "tinybus" -version = "0.1.5" -source = "git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7#d5b949be8ea56eb5a17289a09aa28e8c09bd88b7" -dependencies = [ - "async-trait", - "flate2", - "libc", - "serde", - "serde_json", - "sha2 0.10.9", - "tar", - "tempfile", - "thiserror", - "tokio", - "toml", - "tracing", - "ureq", - "zip", -] - [[package]] name = "tinybus" version = "0.1.5" @@ -2077,7 +2056,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", + "tinybus", "tokio", "tracing", ] @@ -2189,28 +2168,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "tinyflows-import" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "hex", - "serde", - "serde_json", - "sha2 0.11.1", - "tempfile", - "tinyflows", - "tinyflows-bus", - "tinyflows-catalog", - "tinyflows-drivers", - "tinyflows-schedule", - "tinystoragedrivers-core", - "tinystoragedrivers-sqlite", - "tokio", - "tracing", -] - [[package]] name = "tinyflows-module" version = "0.1.0" @@ -2221,7 +2178,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", + "tinybus", "tinybus-module", "tinyflows", "tinyflows-bus", @@ -2231,7 +2188,6 @@ dependencies = [ "tinyflows-schedule", "tinyflows-storage", "tinystoragedrivers-api", - "tinystoragedrivers-busproxy", "tinystoragedrivers-testkit", "tinystoragedrivers-types", "tinytools", @@ -2301,35 +2257,6 @@ dependencies = [ "tinystoragedrivers-types", ] -[[package]] -name = "tinystoragedrivers-bus" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" -dependencies = [ - "serde", - "serde_json", - "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", - "tinystoragedrivers-types", -] - -[[package]] -name = "tinystoragedrivers-busproxy" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" -dependencies = [ - "async-trait", - "bytes", - "futures-core", - "serde", - "serde_json", - "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", - "tinystoragedrivers-api", - "tinystoragedrivers-bus", - "tinystoragedrivers-types", - "tokio", - "zeroize", -] - [[package]] name = "tinystoragedrivers-connect" version = "2.0.0" @@ -2340,22 +2267,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tinystoragedrivers-core" -version = "1.0.0" -source = "git+https://github.com/tinyhumansai/tinystoragedrivers?rev=be94f7568fe7650b0729be55128f48270414b9a7#be94f7568fe7650b0729be55128f48270414b9a7" -dependencies = [ - "async-trait", - "data-encoding", - "serde", - "serde_json", - "sha2 0.10.9", - "thiserror", - "tokio", - "ulid", - "zstd", -] - [[package]] name = "tinystoragedrivers-document" version = "2.0.0" @@ -2413,19 +2324,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "tinystoragedrivers-sqlite" -version = "1.0.0" -source = "git+https://github.com/tinyhumansai/tinystoragedrivers?rev=be94f7568fe7650b0729be55128f48270414b9a7#be94f7568fe7650b0729be55128f48270414b9a7" -dependencies = [ - "rusqlite", - "serde_json", - "sha2 0.10.9", - "tinystoragedrivers-core", - "tokio", - "zstd", -] - [[package]] name = "tinystoragedrivers-testkit" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 977547de..ed8c8345 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,6 @@ async-trait = "0.1" thiserror = "2" tracing = { version = "0.1", default-features = false, features = ["std"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } -rusqlite = { version = "0.40", features = ["bundled"] } sha2 = "0.11" chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "std"] } uuid = { version = "1", features = ["v4", "serde"] } diff --git a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs index d839d8ec..c7985393 100644 --- a/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/ledger_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::drivers::key; use crate::execute::StepRecord; use crate::ledger::{Ledger, Score, conformance}; use tinyflows_storage::Counters; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index ee3f73cb..fe7aef69 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -59,7 +59,10 @@ fn the_schema_declares_the_eight_adaptive_collections_and_the_vault() { "flows.adaptive_steps", "flows.adaptive_workflows", ] { - assert!(names.contains(&name), "{name} missing from {names:?}"); + assert!( + names.iter().any(|candidate| candidate == name), + "{name} missing from {names:?}" + ); } assert!( schema @@ -260,13 +263,19 @@ async fn the_v1_sequences_carry_into_tsd_counters() { ) .await; - assert_eq!(import_legacy_adaptive_counters(&storage).await.unwrap(), 2); + let document_store = docs(&storage); + assert_eq!( + import_legacy_adaptive_counters(&document_store) + .await + .unwrap(), + 2 + ); - let counters = Counters::new(&docs(&storage)); + let counters = Counters::new(&document_store); assert_eq!(counters.peek("flows", "adaptive_rows").await.unwrap(), 41); assert_eq!(counters.peek("flows", "adaptive_lessons").await.unwrap(), 6); assert!( - docs(&storage) + document_store .get("adaptive_counters", "adaptive_rows") .await .unwrap() @@ -281,11 +290,21 @@ async fn the_v1_sequences_carry_into_tsd_counters() { .unwrap(); assert_eq!(id, "ldg_00000042"); // A second run is a no-op. - assert_eq!(import_legacy_adaptive_counters(&storage).await.unwrap(), 0); + assert_eq!( + import_legacy_adaptive_counters(&document_store) + .await + .unwrap(), + 0 + ); } #[tokio::test] async fn importing_with_no_v1_counters_only_records_completion() { let storage = scoped(); - assert_eq!(import_legacy_adaptive_counters(&storage).await.unwrap(), 0); + assert_eq!( + import_legacy_adaptive_counters(&docs(&storage)) + .await + .unwrap(), + 0 + ); } diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index 7ffc46b7..954ff4e6 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -1,9 +1,9 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::workflows::conformance; use serde_json::json; use tinyflows_storage::DocumentStoreExt; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn memory_docs() -> Arc { Arc::clone( diff --git a/crates/tinyflows-adaptive/src/storage.rs b/crates/tinyflows-adaptive/src/storage.rs index 174bbf2f..926d167a 100644 --- a/crates/tinyflows-adaptive/src/storage.rs +++ b/crates/tinyflows-adaptive/src/storage.rs @@ -181,9 +181,7 @@ impl Storage { /// (its agent, its user), so the adaptive tenants live inside it. #[cfg(feature = "storage-drivers")] #[must_use] - pub fn from_documents( - docs: std::sync::Arc, - ) -> Self { + pub fn from_documents(docs: std::sync::Arc) -> Self { Self { ledger: AnyLedger::Driver(crate::drivers::DriverLedger::new(std::sync::Arc::clone( &docs, diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index c71f72a2..661b4c6b 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -1,5 +1,5 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[test] fn memory_has_to_be_asked_for_by_name() { @@ -65,7 +65,7 @@ async fn one_call_scopes_both_halves() { #[cfg(feature = "storage-drivers")] #[tokio::test] async fn one_document_handle_holds_both_halves_and_survives_reopening() { - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let scoped = MemoryStorage::new() .for_scope(&Scope::new("local").expect("valid scope")) .expect("scope"); diff --git a/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs index 770e99bf..a5df9fb8 100644 --- a/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs +++ b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs @@ -74,7 +74,8 @@ async fn a_ledger_and_a_vault_of_different_kinds_drive_the_same_loop() { // either beside the other. Nothing in the loop knows which it got. #[cfg(feature = "storage-drivers")] { - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; + use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; let scoped = MemoryStorage::new() .for_scope(&Scope::new("local").expect("valid scope")) .expect("scope"); @@ -293,4 +294,3 @@ impl LlmProvider for RepairFlow { }) } } - diff --git a/crates/tinyflows-adaptive/tests/support/driver_store.rs b/crates/tinyflows-adaptive/tests/support/driver_store.rs index ac31ff1b..e2eb1843 100644 --- a/crates/tinyflows-adaptive/tests/support/driver_store.rs +++ b/crates/tinyflows-adaptive/tests/support/driver_store.rs @@ -11,7 +11,8 @@ use std::sync::Arc; use tinyflows::store::WorkflowStore; use tinyflows_drivers::store::DriverWorkflowStore; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; /// An empty store over its own in-memory backend. pub fn empty() -> DriverWorkflowStore { diff --git a/crates/tinyflows-bus/src/schedule/mod.rs b/crates/tinyflows-bus/src/schedule/mod.rs index 6ad7b883..a36c1cac 100644 --- a/crates/tinyflows-bus/src/schedule/mod.rs +++ b/crates/tinyflows-bus/src/schedule/mod.rs @@ -261,7 +261,7 @@ pub const STORE_OPERATIONS: &[&str] = &[ /// it accepts a JSON request with `op` and the corresponding core-port fields. /// Operations are ensure/get/put/delete/query/count/delete_where. Replies are /// `{value: JSON}` or `{error: {kind, message}}`, preserving conflict kinds. -/// Core-port JSON uses the tinystoragedrivers-core vocabulary. The host +/// Port JSON uses the tinystoragedrivers-types vocabulary. The host /// delegates mechanically to its scoped DocumentStore and owns no cron policy. pub const DOCUMENT_INTERFACE: &str = "ai.tinyhumans.tinyflows.ScheduleDocuments"; /// Exact document callback member used by the module. diff --git a/crates/tinyflows-drivers/README.md b/crates/tinyflows-drivers/README.md index 0a311bca..a2cab2c5 100644 --- a/crates/tinyflows-drivers/README.md +++ b/crates/tinyflows-drivers/README.md @@ -2,8 +2,10 @@ `tinyflows` storage on [tinystoragedrivers](https://github.com/tinyhumansai/tinystoragedrivers) -document ports, and nothing else: this crate depends on `tinystoragedrivers-core` -only, so no SQLite, MongoDB or file code lives in tinyflows. A host that has +document ports, and nothing else: this crate depends on TinyFlows-owned +`tinyflows-storage` helpers built on the public `tinystoragedrivers-api` and +`tinystoragedrivers-types` contracts, so no SQLite, MongoDB or file code lives +in tinyflows. A host that has opened a storage backend (SQLite on a desktop, MongoDB in the cloud, memory in tests) gets the engine's durable pieces on it, scoped per tenant by the document handle it passes in. @@ -44,20 +46,20 @@ engine nor its expression stack. ## Storage schema v2 -Everything here stores on the uniform schema of `tinystoragedrivers` v1.0.0. +Everything here stores on the uniform schema of `tinystoragedrivers` v2.0.0. `storage_schema()` (owner `flows`) and `cron_storage_schema()` (owner `cron`) -declare the collections and carry the v1 -> v2 steps; the host composes them -with `Registry::compose` and runs `Registry::migrate` per scope, then calls +declare the collections and carry the v1 -> v2 steps; each owning crate runs +its schema migration against the host-bound document port, then calls `import_legacy_kv` and `import_legacy_cron_counters` once per scope (a document -step cannot write `tsd.*`). The graph checkpointer reads and writes the +step cannot write `tsd.*`). All owners share `tsd.migrations` with distinct +owner/item/step progress keys. The graph checkpointer reads and writes the `graph.*` documents of `tinyagents-graph` under namespace `flows`. With the `store` feature `storage_schema()` also declares the workflow store (`flows.workflows`, `flows.workflow_*`). `tinyflows-adaptive` (feature -`storage-drivers`) extends the same schema with `flows.adaptive_*` through -`tinyflows_adaptive::drivers::extend_storage_schema`, and +`storage-drivers`) owns a separate schema for the existing `flows.adaptive_*` +collections and runs its migration on the same bound document port. `tinyflows_adaptive::drivers::import_legacy_adaptive_counters` moves its id -sequences into `tsd.counters`: an owner is declared once, so pass the schema -through both. +sequences into `tsd.counters`. ## The flow catalog (`catalog`) @@ -291,5 +293,7 @@ this crate is the only storage tinyflows has. To bring a host's existing data across, every store has write-if-absent import methods (`FlowCatalogDocuments::import_flow` / `import_run` / ..., `CronDocuments::import_job` / `import_run`, `DriverWorkflowStore::import_workflow` / ...), and the unpublished -`tinyflows-import` crate reads the legacy files and calls them: id for id, never -replacing what is already there, and never touching the legacy files. +The previous one-time file/SQLite importer was retired after the v0.8.9 +transition window. The host owns any remaining legacy file discovery or +compatibility policy; drivers expose document import methods but never read +paths or open files. diff --git a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs index 530cd184..a731f5ef 100644 --- a/crates/tinyflows-drivers/src/catalog/definitions_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/definitions_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::test_support::{catalog, catalog_in, trigger_graph}; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn create_get_list_delete_roundtrip() { @@ -130,7 +130,7 @@ async fn settled_run_updates_its_workflow_summary_only_for_the_winning_status() #[tokio::test] async fn a_run_from_a_removed_flow_incarnation_cannot_stamp_its_replacement() { use crate::catalog::{DEFINITIONS, RUNS, test_support::docs_in}; - use tinyflows_storage::{ Precondition}; + use tinyflows_storage::Precondition; let storage = MemoryStorage::new(); let store = catalog_in(&storage, "incarnation-summary"); diff --git a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs index 55cbc662..2227d342 100644 --- a/crates/tinyflows-drivers/src/catalog/drafts_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/drafts_tests.rs @@ -1,6 +1,6 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::test_support::{catalog, catalog_in}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn graph() -> Value { json!({ "nodes": [{ "id": "t", "kind": "trigger" }], "edges": [] }) diff --git a/crates/tinyflows-drivers/src/catalog/import.rs b/crates/tinyflows-drivers/src/catalog/import.rs index 82bdfdf4..33af9b04 100644 --- a/crates/tinyflows-drivers/src/catalog/import.rs +++ b/crates/tinyflows-drivers/src/catalog/import.rs @@ -10,8 +10,7 @@ //! has since been written to without clobbering anything. The return value says //! whether this call wrote the record. //! -//! The reading half (opening the legacy files) lives in the unpublished -//! `tinyflows-import` crate, which is where the file formats are known. +//! Hosts own any file discovery and source-format decoding. use anyhow::{Context, Result, bail}; use serde_json::{Value, json}; diff --git a/crates/tinyflows-drivers/src/catalog/kv_tests.rs b/crates/tinyflows-drivers/src/catalog/kv_tests.rs index 01a0beab..3a816dbe 100644 --- a/crates/tinyflows-drivers/src/catalog/kv_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/kv_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::test_support::{catalog, catalog_in}; use serde_json::json; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn values_round_trip_per_namespace() { diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index c3dbacd9..a3fe86b3 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -1,14 +1,16 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::FlowCatalogDocuments; use serde_json::json; use tinyflows_storage::{ - DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, - Storage, StorageBackend, WriteOptions, + DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, Storage, StorageBackend, + WriteOptions, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() + MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -54,7 +56,11 @@ fn graph() -> Value { fn the_schema_composes_and_names_every_catalog_collection() { let schema = crate::catalog_storage_schema(); assert_eq!(schema.owner.as_str(), "flows"); - let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); + let names: Vec = schema + .items + .iter() + .map(|(item, _, _)| item.name.clone()) + .collect(); for name in [ "flows.definitions", "flows.revisions", @@ -64,14 +70,24 @@ fn the_schema_composes_and_names_every_catalog_collection() { ] { assert!(names.iter().any(|actual| actual == name), "{name}"); } - assert!(schema.items.iter().all(|(_, _, scope)| *scope == ScopeClass::Tenant)); + assert!( + schema + .items + .iter() + .all(|(_, _, scope)| *scope == ScopeClass::Tenant) + ); assert!( !schema .items .iter() .any(|(item, _, _)| item.name == "flows.workflows") ); - assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); + assert!( + schema + .collections() + .iter() + .any(|item| item.name == "tsd.migrations") + ); } #[tokio::test] diff --git a/crates/tinyflows-drivers/src/catalog/mod.rs b/crates/tinyflows-drivers/src/catalog/mod.rs index 4299f9f4..97d5e9a4 100644 --- a/crates/tinyflows-drivers/src/catalog/mod.rs +++ b/crates/tinyflows-drivers/src/catalog/mod.rs @@ -5,7 +5,7 @@ //! their steps, state, suggestions, drafts), reporting [`FlowUpdateError`] for a //! guarded graph update. The backend is whatever document handle the host //! passes in. It replaces the `flows.db` and draft-file catalog of the retired -//! `tinyflows-sqlite`, whose data `tinyflows-import` brings across. +//! `tinyflows-sqlite`; hosts own any remaining file discovery or conversion. //! //! # Layout (storage schema v2, owner `flows`) //! diff --git a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs index 5565804e..151a30f5 100644 --- a/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/revisions_gap_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use std::sync::Arc; -use tinyflows_storage::{DocumentStore, Precondition}; +use tinyflows_storage::{DocumentStore, Precondition}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; diff --git a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs index da1ff29e..a794b3c1 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_lifecycle_tests.rs @@ -1,10 +1,10 @@ //! A run lifecycle driven through two catalog handles on one backend. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::catalog::test_support::trigger_graph; use tinyflows::nodes::control_flow::dedup::{committed_key, tentative_key}; use tinyflows::nodes::control_flow::dedup_settle; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn step(node: &str) -> FlowRunStep { FlowRunStep { @@ -20,7 +20,7 @@ fn step(node: &str) -> FlowRunStep { async fn two_handles_on_one_backend_agree_on_a_run_lifecycle() { use crate::catalog::FlowStateDocuments; use tinyflows::caps::StateStore; - + let storage = MemoryStorage::new(); let a = crate::catalog::test_support::catalog_in(&storage, "tenant"); let b = crate::catalog::test_support::catalog_in(&storage, "tenant"); @@ -195,4 +195,3 @@ async fn parked_finish_does_not_settle_a_run_claimed_for_resume() { "running" ); } - diff --git a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs index ddb09283..c58170b9 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_recovery_lease_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use tinyflows_catalog::FlowRunStep; -use tinyflows_storage::{ StorageBackend}; +use tinyflows_storage::StorageBackend; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; async fn running_with_lease(lease: &str) -> (FlowCatalogDocuments, String) { let storage = MemoryStorage::new(); diff --git a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs index 301e3281..70eca25d 100644 --- a/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/runs_replace_tests.rs @@ -1,8 +1,8 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use serde_json::json; use std::sync::Arc; -use tinyflows_storage::{DocumentStore, Precondition}; +use tinyflows_storage::{DocumentStore, Precondition}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::catalog::test_support::{Interposed, docs_in, trigger_graph}; diff --git a/crates/tinyflows-drivers/src/catalog/test_support.rs b/crates/tinyflows-drivers/src/catalog/test_support.rs index 0f972c16..19d0794a 100644 --- a/crates/tinyflows-drivers/src/catalog/test_support.rs +++ b/crates/tinyflows-drivers/src/catalog/test_support.rs @@ -1,7 +1,7 @@ //! Fixtures shared by the catalog tests. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows::model::{Node, NodeKind, WorkflowGraph}; use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; @@ -123,9 +123,7 @@ impl DocumentStore for Interposed { &self, collection: &str, id: &str, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { self.maybe_interject("get", collection).await; self.inner.get(collection, id).await } @@ -198,9 +196,7 @@ impl DocumentStore for Interposed { filter: &tinyflows_storage::Filter, sort: &[tinyflows_storage::Sort], patch: &serde_json::Value, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { self.inner.claim(collection, filter, sort, patch).await } @@ -208,4 +204,3 @@ impl DocumentStore for Interposed { self.inner.drop_collection(collection).await } } - diff --git a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs index 8d653d55..449059f5 100644 --- a/crates/tinyflows-drivers/src/checkpoint_import_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_import_tests.rs @@ -1,11 +1,11 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use serde_json::json; use std::sync::Arc; use tinyflows::graph::ids::NodeId; -use tinyflows_storage::{ Scope, StorageBackend}; #[cfg(feature = "catalog")] use tinyflows_storage::{Precondition, WriteOptions}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn legacy_write_import_cas_preserves_existing_task_values() { diff --git a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs index e9ef7490..6b8086c4 100644 --- a/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_layout_tests.rs @@ -4,12 +4,12 @@ //! and from the document shape its `checkpoints_flows_v2` step produces, so a //! drift on either side fails here. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::checkpoint::DriverCheckpointer; use serde_json::json; use std::sync::Arc; use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; -use tinyflows_storage::{DocumentStore, Precondition, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Precondition, Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn docs() -> Arc { Arc::clone( @@ -173,4 +173,3 @@ async fn written_documents_have_exactly_the_tinyagents_fields_and_ids() { assert_eq!(writes.doc["checkpoint_id"], "c1"); assert_eq!(writes.doc["ns"], "flows"); } - diff --git a/crates/tinyflows-drivers/src/checkpoint_tests.rs b/crates/tinyflows-drivers/src/checkpoint_tests.rs index 435c3d89..d51703bd 100644 --- a/crates/tinyflows-drivers/src/checkpoint_tests.rs +++ b/crates/tinyflows-drivers/src/checkpoint_tests.rs @@ -2,12 +2,12 @@ //! store: the checkpointer contract every durable backend is held to, plus the //! isolation and key-encoding guarantees the driver layout adds. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::checkpoint_layout::{checkpoint_id_key, subgraph_key, thread_id_key}; use serde_json::json; use tinyflows::graph::ids::NodeId; -use tinyflows_storage::{ Precondition, Scope, StorageBackend, StorageError}; +use tinyflows_storage::{Precondition, Scope, StorageBackend, StorageError}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( @@ -530,4 +530,3 @@ async fn get_thread_reads_in_insertion_order() { assert_eq!(records[1].state, json!(2)); assert!(cp.get_thread("missing").await.unwrap().is_empty()); } - diff --git a/crates/tinyflows-drivers/src/schedule/admission.rs b/crates/tinyflows-drivers/src/schedule/admission.rs index 61cbbe5c..435eebba 100644 --- a/crates/tinyflows-drivers/src/schedule/admission.rs +++ b/crates/tinyflows-drivers/src/schedule/admission.rs @@ -192,8 +192,7 @@ impl CronDocuments { doc: Value, created_ms: i64, ) -> Result<()> { - let mut options = - tinyflows_storage::WriteOptions::new().schema(super::SCHEMA_VERSION); + let mut options = tinyflows_storage::WriteOptions::new().schema(super::SCHEMA_VERSION); if created_ms > 0 { options = options.created_ms(created_ms); } @@ -305,18 +304,18 @@ impl CronDocuments { _ => false, }; if !owns { - return Err(storage_error( - tinyflows_storage::StorageError::conflict("cron job incarnation changed"), - )); + return Err(storage_error(tinyflows_storage::StorageError::conflict( + "cron job incarnation changed", + ))); } let current = self.docs.get(JOBS, id).await.map_err(storage_error)?; let precondition = match (previous, current.as_ref()) { (None, None) => Precondition::Absent, (Some(old), Some(current)) if old.doc == current.doc => current.unchanged(), _ => { - return Err(storage_error( - tinyflows_storage::StorageError::conflict("cron job changed"), - )); + return Err(storage_error(tinyflows_storage::StorageError::conflict( + "cron job changed", + ))); } }; let op = match next { diff --git a/crates/tinyflows-drivers/src/schedule/admission_tests.rs b/crates/tinyflows-drivers/src/schedule/admission_tests.rs index 5514b192..c340d3dc 100644 --- a/crates/tinyflows-drivers/src/schedule/admission_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/admission_tests.rs @@ -1,10 +1,10 @@ //! Shared admission policy fences both configured and existing handles. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, }; use tinyflows_schedule::CronJobPatch; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn an_existing_unlimited_handle_cannot_bypass_installed_policy() { @@ -324,4 +324,3 @@ async fn policy_installation_reconciles_legacy_rows_written_before_adoption() { a.remove_job(&legacy.id).await.unwrap(); a.add_shell_job(None, daily(), "new slot").await.unwrap(); } - diff --git a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs index 9ca78e4e..452a34ed 100644 --- a/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claim_removal_tests.rs @@ -1,10 +1,10 @@ //! A completed receipt removes only its own unchanged incarnation. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, }; use chrono::Duration; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn completed_receipt_cannot_remove_a_recreated_job() { @@ -139,4 +139,3 @@ async fn later_acquisition_or_edit_invalidates_the_completed_receipt() { assert!(!a.remove_completed_claim(&fresh).await.unwrap()); assert_eq!(a.list_jobs().await.unwrap().len(), 1); } - diff --git a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs index de40c2d4..5ff7951a 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_compaction_tests.rs @@ -1,11 +1,11 @@ //! Occurrence leases are compacted once the job has moved past them, without //! weakening the fencing of stale workers. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{daily, docs_in, store}; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::Schedule; -use tinyflows_storage::{DocumentStoreExt, Query, primitives}; +use tinyflows_storage::{DocumentStoreExt, Query, primitives}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; async fn lease_count(store: &CronDocuments) -> usize { store @@ -148,4 +148,3 @@ async fn a_manual_claim_keeps_its_lease_because_the_schedule_did_not_move() { "the occurrence is still ahead of the job, so its lease record stays" ); } - diff --git a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs index 4c9882ea..ea7d1954 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_safety_tests.rs @@ -1,12 +1,12 @@ //! A claim's lease and the job document are two writes; these pin down that a //! holder which lost its claim cannot commit through the gap between them, and //! that no error path strands a lease. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::CronDocuments; use super::test_support::{FaultDocuments, daily, docs_in}; use chrono::Duration; use std::sync::{Arc, atomic::AtomicBool}; use tinyflows_schedule::CronJobPatch; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn fault_store( backend: &MemoryStorage, @@ -315,4 +315,3 @@ async fn a_failed_lease_release_after_a_committed_completion_does_not_fail_it() assert!(store.list_runs(&job.id, 10).await.unwrap().is_empty()); assert!(store.find_job(&job.id).await.unwrap().is_none()); } - diff --git a/crates/tinyflows-drivers/src/schedule/claims_tests.rs b/crates/tinyflows-drivers/src/schedule/claims_tests.rs index 59cb5c91..4389d024 100644 --- a/crates/tinyflows-drivers/src/schedule/claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/claims_tests.rs @@ -1,10 +1,10 @@ //! Occurrences are leased across handles and fenced against stale workers. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::{ CronDocuments, test_support::{daily, docs_in}, }; use chrono::Duration; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn two_handles_cannot_claim_the_same_occurrence() { @@ -489,4 +489,3 @@ async fn a_claim_is_a_tsd_lease_and_run_numbers_a_tsd_counter() { .unwrap() ); } - diff --git a/crates/tinyflows-drivers/src/schedule/import.rs b/crates/tinyflows-drivers/src/schedule/import.rs index d4e6fdfb..993138a8 100644 --- a/crates/tinyflows-drivers/src/schedule/import.rs +++ b/crates/tinyflows-drivers/src/schedule/import.rs @@ -19,8 +19,8 @@ //! an import into a scope with history never overwrites it. The counter is //! always left beyond every number used. //! -//! The reading half (opening `jobs.db`) lives in the unpublished -//! `tinyflows-import` crate. +//! Hosts own any file discovery; these methods only write through the +//! caller-provided storage port. use anyhow::{Context, Result}; use tinyflows_schedule::{CronJob, CronRun, JobType}; diff --git a/crates/tinyflows-drivers/src/schedule/import_tests.rs b/crates/tinyflows-drivers/src/schedule/import_tests.rs index 875d5e5c..682f1daa 100644 --- a/crates/tinyflows-drivers/src/schedule/import_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/import_tests.rs @@ -1,6 +1,6 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use crate::schedule::job_not_found; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::schedule::test_support::{channel_origin, daily, docs_in, store}; use chrono::{TimeZone, Utc}; diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs index 1dd0ecab..1e7fc7b7 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_fence_tests.rs @@ -1,4 +1,3 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use async_trait::async_trait; use chrono::{Duration, Utc}; @@ -6,9 +5,10 @@ use serde_json::Value; use std::sync::Arc; use tinyflows_schedule::Schedule; use tinyflows_storage::{ - Capabilities, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, - Scope, Sort, StorageBackend, Version, Versioned, WriteOp, WriteResult, + Capabilities, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Scope, Sort, + StorageBackend, Version, Versioned, WriteOp, WriteResult, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tokio::sync::Notify; #[derive(Debug)] @@ -25,10 +25,7 @@ impl DocumentStore for HeldClaimWrite { self.inner.capabilities() } - async fn ensure_collection( - &self, - spec: &CollectionSpec, - ) -> tinyflows_storage::Result<()> { + async fn ensure_collection(&self, spec: &CollectionSpec) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -67,11 +64,7 @@ impl DocumentStore for HeldClaimWrite { self.inner.query(collection, query).await } - async fn count( - &self, - collection: &str, - filter: &Filter, - ) -> tinyflows_storage::Result { + async fn count(&self, collection: &str, filter: &Filter) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } @@ -93,10 +86,7 @@ impl DocumentStore for HeldClaimWrite { self.inner.claim(collection, filter, sort, patch).await } - async fn atomic_batch( - &self, - ops: Vec, - ) -> tinyflows_storage::Result> { + async fn atomic_batch(&self, ops: Vec) -> tinyflows_storage::Result> { let late_claim = ops.iter().any(|op| match op { WriteOp::Put { collection, doc, .. @@ -140,7 +130,9 @@ impl DocumentStore for HeldClaimWrite { #[tokio::test] async fn admission_fence_rejects_claim_transaction_that_commits_after_reconciliation() { let backend = MemoryStorage::new(); - let scoped = backend.for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = backend + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let held_docs = Arc::new(HeldClaimWrite { inner: scoped.documents().clone(), entered: Notify::new(), @@ -208,7 +200,9 @@ async fn admission_fence_rejects_claim_transaction_that_commits_after_reconcilia #[tokio::test] async fn reconciliation_does_not_fence_changed_snapshots_and_lost_fence_reply_is_idempotent() { let backend = MemoryStorage::new(); - let scoped = backend.for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = backend + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let held_docs = Arc::new(HeldClaimWrite { inner: scoped.documents().clone(), entered: Notify::new(), diff --git a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs index 7917af49..812c1eea 100644 --- a/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/manual_claim_tests.rs @@ -1,8 +1,8 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::docs_in; use super::*; use chrono::{Duration, Utc}; use tinyflows_schedule::{CronJobPatch, DeliveryStatus, Schedule}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn reconciles_manual_claim_receipt_after_an_ambiguous_claim_reply() { diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index bf2cd988..878ef8ba 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -1,15 +1,16 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use chrono::TimeZone; use serde_json::json; use tinyflows_schedule::Schedule; use tinyflows_storage::{ - DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, - Storage, StorageBackend, + DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, Storage, StorageBackend, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() + MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { @@ -46,9 +47,18 @@ fn v1_job(schedule: Value) -> Value { fn the_schema_composes_and_declares_the_cron_collections() { let schema = storage_schema(); assert_eq!(schema.owner.as_str(), "cron"); - let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); + let names: Vec = schema + .items + .iter() + .map(|(item, _, _)| item.name.clone()) + .collect(); assert_eq!(names, ["cron.jobs", "cron.runs", "cron.admission"]); - assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); + assert!( + schema + .collections() + .iter() + .any(|item| item.name == "tsd.migrations") + ); } #[tokio::test] @@ -88,9 +98,15 @@ async fn v1_jobs_runs_and_the_ledger_migrate_and_read_back() { seed(&storage, "cron_counters", "runs", json!({"next": 8})).await; migrate(&storage).await; - assert!(import_legacy_cron_counters(storage.documents()).await.unwrap()); assert!( - !import_legacy_cron_counters(storage.documents()).await.unwrap(), + import_legacy_cron_counters(storage.documents()) + .await + .unwrap() + ); + assert!( + !import_legacy_cron_counters(storage.documents()) + .await + .unwrap(), "runs once" ); diff --git a/crates/tinyflows-drivers/src/schedule/mod.rs b/crates/tinyflows-drivers/src/schedule/mod.rs index 5022ce69..09bbe190 100644 --- a/crates/tinyflows-drivers/src/schedule/mod.rs +++ b/crates/tinyflows-drivers/src/schedule/mod.rs @@ -4,7 +4,7 @@ //! storage backend (SQLite on a desktop, MongoDB in the cloud, memory in tests) //! hands it a document handle, usually one already bound to a tenant scope. It //! replaces the `jobs.db` store of the retired `tinyflows-sqlite`, whose data -//! `tinyflows-import` brings across. +//! host-owned migration tooling may bring across. //! //! # Layout (storage schema v2, owner `cron`) //! @@ -241,8 +241,7 @@ async fn compare_job( .await { Ok(results) => { - let Some(tinyflows_storage::WriteResult::Put { version }) = results.first() - else { + let Some(tinyflows_storage::WriteResult::Put { version }) = results.first() else { anyhow::bail!("cron store: invalid atomic write result"); }; return Ok(Some(Versioned { diff --git a/crates/tinyflows-drivers/src/schedule/mod_tests.rs b/crates/tinyflows-drivers/src/schedule/mod_tests.rs index dae72496..d46cae1b 100644 --- a/crates/tinyflows-drivers/src/schedule/mod_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/mod_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{daily, docs_in, store}; use chrono::Utc; use serde_json::json; -use tinyflows_storage::{ Precondition}; +use tinyflows_storage::Precondition; #[test] fn limits_default_like_the_sqlite_store_and_clamp_to_one() { diff --git a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs index d4cdf2ab..cb302624 100644 --- a/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/removal_claims_tests.rs @@ -1,10 +1,10 @@ //! Removal compatibility and admission bookkeeping across supported backends. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::super::{ CronDocuments, JOBS, test_support::{FaultDocuments, daily, docs_in}, }; use std::sync::{Arc, atomic::AtomicBool}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] async fn nontransactional_deduplication_and_clear_keep_legacy_behavior() { @@ -135,4 +135,3 @@ async fn a_legacy_row_without_enabled_is_imported_as_disabled() { b.add_shell_job(None, daily(), "enabled").await.unwrap(); assert_eq!(b.list_jobs().await.unwrap().len(), 2); } - diff --git a/crates/tinyflows-drivers/src/schedule/test_support.rs b/crates/tinyflows-drivers/src/schedule/test_support.rs index e5187cf8..4628a21b 100644 --- a/crates/tinyflows-drivers/src/schedule/test_support.rs +++ b/crates/tinyflows-drivers/src/schedule/test_support.rs @@ -1,7 +1,7 @@ //! Fixtures the schedule tests share. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows_schedule::{JobOrigin, Schedule}; use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; @@ -99,9 +99,7 @@ impl DocumentStore for FaultDocuments { &self, c: &str, id: &str, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { if c == "cron.jobs" { let read = self .faults @@ -180,9 +178,7 @@ impl DocumentStore for FaultDocuments { f: &tinyflows_storage::Filter, s: &[tinyflows_storage::Sort], p: &serde_json::Value, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { self.inner.claim(c, f, s, p).await } async fn drop_collection(&self, c: &str) -> tinyflows_storage::Result<()> { @@ -205,4 +201,3 @@ impl DocumentStore for FaultDocuments { self.inner.atomic_batch(ops).await } } - diff --git a/crates/tinyflows-drivers/src/state_store_tests.rs b/crates/tinyflows-drivers/src/state_store_tests.rs index 0343dd6a..3095caf5 100644 --- a/crates/tinyflows-drivers/src/state_store_tests.rs +++ b/crates/tinyflows-drivers/src/state_store_tests.rs @@ -1,7 +1,7 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use serde_json::json; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn docs(storage: &MemoryStorage, scope: &str) -> Arc { Arc::clone( diff --git a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs index 1db945bc..04d5a28d 100644 --- a/crates/tinyflows-drivers/src/storage/kv_import_tests.rs +++ b/crates/tinyflows-drivers/src/storage/kv_import_tests.rs @@ -1,10 +1,12 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::*; use serde_json::json; use tinyflows_storage::{Kv, Scope, Storage, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() + MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap() } async fn seed(storage: &Storage, collection: &str, id: &str, body: Value) { diff --git a/crates/tinyflows-drivers/src/store/authoring_tests.rs b/crates/tinyflows-drivers/src/store/authoring_tests.rs index 66dfd7d0..344946ed 100644 --- a/crates/tinyflows-drivers/src/store/authoring_tests.rs +++ b/crates/tinyflows-drivers/src/store/authoring_tests.rs @@ -3,8 +3,8 @@ //! The theme is that a bad edit costs an error message, never the saved //! workflow. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::graph_ops::GraphOp; @@ -322,4 +322,3 @@ fn editing_a_workflow_that_does_not_exist_says_so() { assert!(matches!(err, WorkflowError::NotFound(_)), "got {err:?}"); } - diff --git a/crates/tinyflows-drivers/src/store/concurrency_tests.rs b/crates/tinyflows-drivers/src/store/concurrency_tests.rs index 8bbf416f..984f4137 100644 --- a/crates/tinyflows-drivers/src/store/concurrency_tests.rs +++ b/crates/tinyflows-drivers/src/store/concurrency_tests.rs @@ -2,8 +2,8 @@ //! captured beside it must hold up when two threads, or two store instances, //! write at once. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::{Arc, Barrier}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows::store::WorkflowStore; @@ -189,4 +189,3 @@ fn a_guarded_save_of_an_invalid_graph_is_refused_before_anything_is_written() { ); assert!(store.list_revisions("guarded").unwrap().is_empty()); } - diff --git a/crates/tinyflows-drivers/src/store/history_tests.rs b/crates/tinyflows-drivers/src/store/history_tests.rs index 463ecd40..40027eae 100644 --- a/crates/tinyflows-drivers/src/store/history_tests.rs +++ b/crates/tinyflows-drivers/src/store/history_tests.rs @@ -3,10 +3,10 @@ //! restores exactly that revision, and history never leaks into the ordinary //! workflow listing. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{record, store, store_over}; use tinyflows::store::types::WorkflowError; use tinyflows::store::{MAX_REVISIONS, WorkflowStore, require, rollback, undo_last}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[test] fn a_workflow_saved_for_the_first_time_has_no_history_to_go_back_to() { @@ -211,4 +211,3 @@ fn history_does_not_show_up_in_the_workflow_listing() { assert_eq!(store.list().unwrap().len(), 1); assert_eq!(require(&store, "greet").unwrap().description, "again"); } - diff --git a/crates/tinyflows-drivers/src/store/import.rs b/crates/tinyflows-drivers/src/store/import.rs index 7a89e60d..ee23d810 100644 --- a/crates/tinyflows-drivers/src/store/import.rs +++ b/crates/tinyflows-drivers/src/store/import.rs @@ -9,8 +9,7 @@ //! that has since been written to, without clobbering anything. The return //! value says whether this call wrote the record. //! -//! The reading half (opening the legacy files) lives in the unpublished -//! `tinyflows-import` crate, which is where the file formats are known. +//! Hosts own any file discovery and source-format decoding. use serde_json::Value; use tinyflows::store::types::{ diff --git a/crates/tinyflows-drivers/src/store/import_tests.rs b/crates/tinyflows-drivers/src/store/import_tests.rs index 73f28561..2ce2550a 100644 --- a/crates/tinyflows-drivers/src/store/import_tests.rs +++ b/crates/tinyflows-drivers/src/store/import_tests.rs @@ -1,10 +1,10 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{ NoteKind, NoteSource, ProposalStatus, RunStatus, WorkflowError, WorkflowNote, WorkflowProposal, WorkflowRevision, }; use tinyflows::store::{MAX_NOTES, MAX_REVISIONS, WorkflowStore, mint_note_id, new_run_record}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use crate::store::test_support::{record, store, store_over}; diff --git a/crates/tinyflows-drivers/src/store/locks_tests.rs b/crates/tinyflows-drivers/src/store/locks_tests.rs index f9079d49..f9abf153 100644 --- a/crates/tinyflows-drivers/src/store/locks_tests.rs +++ b/crates/tinyflows-drivers/src/store/locks_tests.rs @@ -1,13 +1,13 @@ //! The decision lease: exclusion across store instances, release on drop, and //! recovery from a holder that went away. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::mpsc; use std::time::Duration; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::WorkflowStore; -use tinyflows_storage::{Key, Precondition, primitives}; +use tinyflows_storage::{Key, Precondition, primitives}; use super::DriverWorkflowStore; use super::locks::{LEASE_OWNER, lease_key}; @@ -146,4 +146,3 @@ fn a_lease_id_that_is_not_usable_is_refused() { let store = store_over(&MemoryStorage::new(), "local"); assert!(store.lock_proposal_decision("").is_err()); } - diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index 6d6fbac4..c3eac5b9 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -1,17 +1,19 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::{Value, json}; use tinyflows::store::WorkflowStore; use tinyflows_storage::{ - DocumentStore, DocumentStoreExt, MigrateOptions, Precondition, Query, - Scope, ScopeClass, Storage, StorageBackend, + DocumentStore, DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, ScopeClass, + Storage, StorageBackend, }; use super::*; fn scoped() -> Storage { - MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap() + MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap() } fn docs(storage: &Storage) -> Arc { @@ -43,7 +45,11 @@ fn v1_record(id: &str, description: &str) -> String { fn the_schema_names_every_workflow_store_collection() { let schema = crate::storage_schema(); assert_eq!(schema.owner.as_str(), "flows"); - let names: Vec = schema.items.iter().map(|(item, _, _)| item.name.clone()).collect(); + let names: Vec = schema + .items + .iter() + .map(|(item, _, _)| item.name.clone()) + .collect(); for name in [ "flows.workflows", "flows.workflow_revisions", @@ -52,10 +58,23 @@ fn the_schema_names_every_workflow_store_collection() { "flows.workflow_proposals", "flows.workflow_quarantine", ] { - assert!(names.iter().any(|actual| actual == name), "{name} missing from {names:?}"); + assert!( + names.iter().any(|actual| actual == name), + "{name} missing from {names:?}" + ); } - assert!(schema.items.iter().all(|(_, _, scope)| *scope == ScopeClass::Tenant)); - assert!(schema.collections().iter().any(|item| item.name == "tsd.migrations")); + assert!( + schema + .items + .iter() + .all(|(_, _, scope)| *scope == ScopeClass::Tenant) + ); + assert!( + schema + .collections() + .iter() + .any(|item| item.name == "tsd.migrations") + ); } #[tokio::test] diff --git a/crates/tinyflows-drivers/src/store/mod_tests.rs b/crates/tinyflows-drivers/src/store/mod_tests.rs index 4957f3df..d9768694 100644 --- a/crates/tinyflows-drivers/src/store/mod_tests.rs +++ b/crates/tinyflows-drivers/src/store/mod_tests.rs @@ -2,13 +2,13 @@ //! `defaults` block round-trips, an uncompilable graph is refused, ids are //! guarded, and the host policy judges reads as well as writes. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{WorkflowDefaults, WorkflowError, WorkflowRecord}; use tinyflows::store::{HostPolicy, WorkflowStore, parse_workflow, require}; -use tinyflows_storage::{ Precondition}; +use tinyflows_storage::Precondition; use super::test_support::{Faulty, block_on, docs_in, record, store, valid_document}; use super::*; @@ -337,4 +337,3 @@ fn debug_names_the_store_without_dumping_the_handle() { fn the_journal_cap_is_the_engines() { assert_eq!(JOURNAL_CAP, tinyflows::store::MAX_NOTES); } - diff --git a/crates/tinyflows-drivers/src/store/notes_tests.rs b/crates/tinyflows-drivers/src/store/notes_tests.rs index 600f7e8b..72bf7e75 100644 --- a/crates/tinyflows-drivers/src/store/notes_tests.rs +++ b/crates/tinyflows-drivers/src/store/notes_tests.rs @@ -2,11 +2,11 @@ //! happens when things go wrong: a journal that cannot be parsed, an id that is //! not usable, concurrent appenders. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{NoteKind, NoteSource, WorkflowError, WorkflowNote}; use tinyflows::store::{MAX_NOTES, WorkflowStore, current_notes, mint_note_id}; -use tinyflows_storage::{DocumentStoreExt, Precondition, Query}; +use tinyflows_storage::{DocumentStoreExt, Precondition, Query}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{block_on, docs_in, store, store_over}; use super::{DriverWorkflowStore, NOTES, QUARANTINE}; @@ -287,4 +287,3 @@ fn a_note_round_trips_every_field() { store.append_note(&written).unwrap(); assert_eq!(store.list_notes("sweep").unwrap().remove(0), written); } - diff --git a/crates/tinyflows-drivers/src/store/proposals_tests.rs b/crates/tinyflows-drivers/src/store/proposals_tests.rs index 87fc5c93..cb22495e 100644 --- a/crates/tinyflows-drivers/src/store/proposals_tests.rs +++ b/crates/tinyflows-drivers/src/store/proposals_tests.rs @@ -4,13 +4,13 @@ //! proposals, a decision that replaces rather than adds, a save that refuses //! when the graph moved underneath it. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::atomic::Ordering; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::types::{ProposalStatus, WorkflowError, WorkflowProposal, fingerprint}; use tinyflows::store::{WorkflowStore, mint_proposal_id, require_proposal}; -use tinyflows_storage::{DocumentStoreExt, Precondition}; +use tinyflows_storage::{DocumentStoreExt, Precondition}; use super::test_support::{Faulty, block_on, docs_in, record, store, store_over}; use super::{DEFINITIONS, DriverWorkflowStore, PROPOSALS}; @@ -285,4 +285,3 @@ fn an_edit_that_lands_between_the_check_and_the_write_refuses_the_proposal() { .is_empty() ); } - diff --git a/crates/tinyflows-drivers/src/store/runs_tests.rs b/crates/tinyflows-drivers/src/store/runs_tests.rs index b85f71f9..6f08bb37 100644 --- a/crates/tinyflows-drivers/src/store/runs_tests.rs +++ b/crates/tinyflows-drivers/src/store/runs_tests.rs @@ -1,10 +1,10 @@ //! Run-record listing: newest first, scoped to their workflow, and a run that //! was never recorded is a distinct error rather than a silent `None`. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::test_support::{block_on, store}; use tinyflows::store::types::{RunExecutor, RunStatus, WorkflowError}; use tinyflows::store::{WorkflowStore, new_run_record, require_run}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[test] fn runs_are_listed_newest_first_and_scoped_to_their_workflow() { @@ -127,7 +127,7 @@ fn an_executor_and_a_cancel_request_survive_a_write_and_read() { #[test] fn an_unreadable_run_document_is_skipped_by_listings_and_named_by_a_direct_read() { use serde_json::json; - use tinyflows_storage::{ Precondition}; + use tinyflows_storage::Precondition; let storage = MemoryStorage::new(); let docs = super::test_support::docs_in(&storage, "local"); let store = super::DriverWorkflowStore::new(docs.clone()); @@ -154,4 +154,3 @@ fn an_unreadable_run_document_is_skipped_by_listings_and_named_by_a_direct_read( WorkflowError::Malformed(_) )); } - diff --git a/crates/tinyflows-drivers/src/store/test_support.rs b/crates/tinyflows-drivers/src/store/test_support.rs index 3967be6b..9f3e4119 100644 --- a/crates/tinyflows-drivers/src/store/test_support.rs +++ b/crates/tinyflows-drivers/src/store/test_support.rs @@ -1,7 +1,7 @@ //! Fixtures the store tests share. -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows::store::parse_workflow; @@ -124,9 +124,7 @@ impl DocumentStore for Faulty { &self, collection: &str, id: &str, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { self.inner.get(collection, id).await } @@ -206,9 +204,7 @@ impl DocumentStore for Faulty { filter: &tinyflows_storage::Filter, sort: &[tinyflows_storage::Sort], patch: &serde_json::Value, - ) -> tinyflows_storage::Result< - Option>, - > { + ) -> tinyflows_storage::Result>> { self.inner.claim(collection, filter, sort, patch).await } @@ -241,4 +237,3 @@ impl DocumentStore for Faulty { self.inner.atomic_batch_checked(checks, ops).await } } - diff --git a/crates/tinyflows-import/Cargo.toml b/crates/tinyflows-import/Cargo.toml deleted file mode 100644 index 0ac48c45..00000000 --- a/crates/tinyflows-import/Cargo.toml +++ /dev/null @@ -1,67 +0,0 @@ -# Reads the stores tinyflows hosts used before storage moved to the -# `tinystoragedrivers` ports, and writes what they held into the port-backed -# stores (`tinyflows-drivers`), id for id and only where nothing exists yet. -# -# A crate of its own, and not published: it names the storage crates by git tag -# (as `tinyflows-drivers` does), and it is the one place that still knows the -# retired formats (`flows.db`, `jobs.db`, `checkpoints.db`, the JSON/JSONL -# stores). The SQLite files are read through the SQLite driver's native handle, -# never a database client of this crate's own. -[package] -name = "tinyflows-import" -version = "0.1.0" -description = "One-time, read-only import of legacy tinyflows stores into the storage-port stores" -edition.workspace = true -rust-version = "1.88" -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false - -[features] -# Every importer by default. A host builds with `default-features = false` and -# only the importers it ships: a build that runs scheduled jobs but no flows -# takes `schedule` alone and pulls neither the engine nor its expression stack. -default = ["engine", "store", "catalog", "schedule"] -# `import_sqlite_checkpoints`, `import_file_checkpoints`. -engine = ["checkpoint-read", "checkpoint-write"] -# `import_workflow_store`. -store = ["engine", "tinyflows/store", "tinyflows-drivers/store"] -# `import_flow_catalog`, `import_file_state`. -catalog = ["engine", "flow-write", "flow-read", "dep:tinyflows-catalog", "dep:tinyflows-drivers", "tinyflows-drivers/catalog"] -# `import_cron`. -schedule = ["schedule-read", "dep:tinyflows-schedule", "dep:tinyflows-drivers", "tinyflows-drivers/schedule"] -# `read_legacy_cron`: reads `jobs.db` into bus DTOs without linking the -# scheduler implementation or cron store. -schedule-read = ["dep:tinyflows-bus", "dep:tinystoragedrivers-sqlite"] -# Host-side legacy flow definitions/revisions/runs/state/suggestions/drafts. -# Links bus and SQLite reader only, no TinyFlows implementation crates. -flow-read = ["dep:tinyflows-bus", "dep:tinystoragedrivers-sqlite"] -# Shared semantic writer used by the public API and owning Catalog module. -flow-write = ["checkpoint-write", "dep:tinyflows-catalog", "tinyflows-drivers/catalog"] -# Host-side SQLite and JSONL checkpoint/write readers; implementation-free. -checkpoint-read = ["flow-read"] -# Shared checkpoint decode/replay logic. This does not pull in the catalog API. -checkpoint-write = ["dep:tinyflows", "dep:tinyflows-drivers", "tinyflows-drivers/engine", "dep:tinyflows-bus"] - -[dependencies] -tinyflows = { workspace = true, optional = true } -tinyflows-catalog = { version = "0.1.0", path = "../tinyflows-catalog", optional = true } -tinyflows-bus = { path = "../tinyflows-bus", optional = true } -tinyflows-schedule = { version = "0.1.0", path = "../tinyflows-schedule", optional = true } -tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, optional = true } -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7" } -# The SQLite driver's native handle: how the legacy `.db` files are opened. It -# re-exports the `rusqlite` its handle hands out, which is all this crate uses. -tinystoragedrivers-sqlite = { git = "https://github.com/tinyhumansai/tinystoragedrivers", rev = "be94f7568fe7650b0729be55128f48270414b9a7", optional = true } -anyhow = "1.0" -chrono = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -sha2 = { workspace = true } -hex = "0.4" -tempfile = "3" -tracing = { workspace = true } - -[dev-dependencies] -tokio = { workspace = true } diff --git a/crates/tinyflows-import/src/catalog.rs b/crates/tinyflows-import/src/catalog.rs deleted file mode 100644 index cc552967..00000000 --- a/crates/tinyflows-import/src/catalog.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! The public legacy catalog API, composed from the raw host reader and the -//! shared module-side semantic writer. - -use std::path::Path; -use std::sync::{Arc, Mutex}; - -use anyhow::{Context, Result}; -use tinyflows_drivers::catalog::FlowCatalogDocuments; - -use crate::report::ImportReport; - -/// Imports the retired `flows.db` and `drafts/*.json` sources into `catalog`. -/// -/// The source reader emits bounded raw DTO batches. The same semantic writer -/// used by Catalog v6 performs graph migration, validation, idempotent writes -/// and report accounting, preserving the behavior of the established API. -pub async fn import_flow_catalog( - dir: &Path, - catalog: &FlowCatalogDocuments, -) -> Result { - let report = Arc::new(Mutex::new(ImportReport::default())); - let report_for_send = Arc::clone(&report); - let mut send = move |records: Vec| { - let report = Arc::clone(&report_for_send); - async move { - let response = crate::legacy_batch::apply_legacy_batch(&records, catalog, None).await?; - report - .lock() - .map_err(|_| anyhow::anyhow!("legacy import report lock poisoned"))? - .merge_legacy_batch(response); - Ok(()) - } - }; - crate::flow_reader::read_legacy_flow_catalog(dir, &mut send) - .await - .with_context(|| format!("importing legacy flow catalog from {}", dir.display()))?; - drop(send); - Arc::try_unwrap(report) - .map_err(|_| anyhow::anyhow!("legacy import report still has active readers"))? - .into_inner() - .map_err(|_| anyhow::anyhow!("legacy import report lock poisoned")) -} - -#[cfg(test)] -#[path = "catalog_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/catalog_tests.rs b/crates/tinyflows-import/src/catalog_tests.rs deleted file mode 100644 index ed58abda..00000000 --- a/crates/tinyflows-import/src/catalog_tests.rs +++ /dev/null @@ -1,328 +0,0 @@ -use super::*; -use crate::test_support::{docs, expected, fixtures, json, sorted_by, tree_digest}; -use serde_json::json; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -fn catalog_dir() -> std::path::PathBuf { - fixtures().join("catalog") -} - -fn target() -> FlowCatalogDocuments { - FlowCatalogDocuments::new(docs()) -} - -#[tokio::test] -async fn the_whole_catalog_comes_across_with_its_ids_and_every_field() { - let catalog = target(); - let report = import_flow_catalog(&catalog_dir(), &catalog).await.unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - let expected = expected(); - let expected = &expected["catalog"]; - - // Flows. - let (flows, skipped) = catalog.list_flows().await.unwrap(); - assert_eq!(skipped, 0); - assert_eq!( - sorted_by(flows.iter().map(json).collect(), "id"), - sorted_by(expected["flows"].as_array().unwrap().clone(), "id") - ); - // Revisions and runs, per flow. - for flow in &flows { - let revisions = catalog.list_revisions(&flow.id, 100).await.unwrap(); - assert_eq!( - sorted_by(revisions.iter().map(json).collect(), "id"), - sorted_by( - expected["revisions"][&flow.id].as_array().unwrap().clone(), - "id" - ), - "revisions of {}", - flow.id - ); - let runs = catalog.list_flow_runs(&flow.id, 100).await.unwrap(); - assert_eq!( - sorted_by(runs.iter().map(json).collect(), "id"), - sorted_by(expected["runs"][&flow.id].as_array().unwrap().clone(), "id"), - "runs of {}", - flow.id - ); - } - // State, suggestions, drafts. - for entry in expected["kv"].as_array().unwrap() { - let (namespace, key, value) = ( - entry[0].as_str().unwrap(), - entry[1].as_str().unwrap(), - &entry[2], - ); - assert_eq!( - catalog.kv_get(namespace, key).await.unwrap().as_ref(), - Some(value), - "{namespace}/{key}" - ); - } - let suggestions = catalog.list_suggestions(None, 100).await.unwrap(); - assert_eq!( - sorted_by(suggestions.iter().map(json).collect(), "id"), - sorted_by(expected["suggestions"].as_array().unwrap().clone(), "id") - ); - let drafts = catalog.list_drafts().await.unwrap(); - assert_eq!( - sorted_by(drafts.iter().map(json).collect(), "id"), - sorted_by(expected["drafts"].as_array().unwrap().clone(), "id") - ); -} - -#[tokio::test] -async fn the_report_counts_every_kind() { - let report = import_flow_catalog(&catalog_dir(), &target()) - .await - .unwrap(); - for (kind, written) in [ - ("flows", 2), - ("revisions", 2), - ("runs", 4), - ("state", 3), - ("suggestions", 2), - ("drafts", 2), - ] { - assert_eq!(report.tally(kind).written, written, "{kind}"); - assert_eq!(report.tally(kind).existing, 0, "{kind}"); - } - assert_eq!(report.total_written(), 15); -} - -#[tokio::test] -async fn a_dismissed_suggestion_stays_dismissed_and_a_parked_run_stays_parked() { - let catalog = target(); - import_flow_catalog(&catalog_dir(), &catalog).await.unwrap(); - let suggestions = catalog.list_suggestions(None, 100).await.unwrap(); - let dismissed = suggestions.iter().find(|s| s.id == "sugg-2").unwrap(); - assert_eq!(dismissed.status.as_str(), "dismissed"); - let parked = catalog.get_flow_run("run-parked").await.unwrap().unwrap(); - assert_eq!(parked.status, "pending_approval"); - assert_eq!(parked.pending_approvals, ["gate"]); - assert_eq!(parked.graph_hash.as_deref(), Some("hash-1")); - let live = catalog.get_flow_run("run-live").await.unwrap().unwrap(); - assert_eq!(live.status, "running"); -} - -#[tokio::test] -async fn importing_again_writes_nothing_and_finds_everything_already_there() { - let catalog = target(); - let first = import_flow_catalog(&catalog_dir(), &catalog).await.unwrap(); - let second = import_flow_catalog(&catalog_dir(), &catalog).await.unwrap(); - assert_eq!(second.total_written(), 0); - for (kind, tally) in first.kinds() { - assert_eq!(second.tally(kind).existing, tally.written, "{kind}"); - } -} - -#[tokio::test] -async fn the_legacy_files_are_left_exactly_as_they_were() { - let before = tree_digest(&fixtures()); - import_flow_catalog(&catalog_dir(), &target()) - .await - .unwrap(); - assert_eq!(tree_digest(&fixtures()), before); -} - -#[tokio::test] -async fn nothing_the_target_already_holds_is_replaced() { - let catalog = target(); - let live = catalog - .create_flow( - "live name".into(), - serde_json::from_value(json!({ "nodes": [], "edges": [] })).unwrap(), - false, - true, - ) - .await - .unwrap(); - // The live flow takes the legacy flow's id. - catalog.remove_flow(&live.id).await.unwrap(); - let mut owned = live.clone(); - owned.id = "flow-alpha".into(); - catalog.upsert_flow(&owned).await.unwrap(); - catalog - .kv_set("workflow:flow-alpha", "cursor", &json!("mine")) - .await - .unwrap(); - - let report = import_flow_catalog(&catalog_dir(), &catalog).await.unwrap(); - - assert_eq!(report.tally("flows").existing, 1); - assert_eq!(report.tally("flows").written, 1); - assert_eq!( - catalog.get_flow("flow-alpha").await.unwrap().unwrap().name, - "live name" - ); - assert_eq!( - catalog - .kv_get("workflow:flow-alpha", "cursor") - .await - .unwrap(), - Some(json!("mine")) - ); - assert_eq!(report.tally("state").existing, 1); -} - -#[tokio::test] -async fn a_directory_with_nothing_in_it_is_an_empty_report() { - let dir = tempfile::tempdir().unwrap(); - let report = import_flow_catalog(dir.path(), &target()).await.unwrap(); - assert_eq!(report.total_written(), 0); - assert!(report.is_clean()); - let report = import_flow_catalog(&dir.path().join("never-existed"), &target()) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); -} - -fn legacy_db(dir: &std::path::Path, sql: &str) { - Connection::open(dir.join("flows.db")) - .unwrap() - .execute_batch(sql) - .unwrap(); -} - -const GRAPH: &str = r#"{"nodes":[{"id":"t","kind":"trigger","name":"start","config":{"trigger_kind":"manual"}}],"edges":[]}"#; - -#[tokio::test] -async fn a_database_from_before_later_columns_reads_with_their_defaults() { - let dir = tempfile::tempdir().unwrap(); - legacy_db( - dir.path(), - &format!( - "CREATE TABLE flow_definitions (id TEXT PRIMARY KEY, name TEXT NOT NULL, graph_json TEXT NOT NULL, - enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - last_run_at TEXT, last_status TEXT); - INSERT INTO flow_definitions VALUES ('old', 'Old', '{GRAPH}', 1, '2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z', NULL, NULL); - CREATE TABLE flow_runs (id TEXT PRIMARY KEY, flow_id TEXT NOT NULL, thread_id TEXT NOT NULL, status TEXT NOT NULL, - started_at TEXT NOT NULL, finished_at TEXT, steps_json TEXT NOT NULL DEFAULT '[]', - pending_approvals_json TEXT NOT NULL DEFAULT '[]', error TEXT); - INSERT INTO flow_runs VALUES ('r', 'old', 't', 'completed', '2024-01-01T00:00:00Z', '2024-01-01T00:00:01Z', '[]', '[]', NULL);" - ), - ); - let catalog = target(); - let report = import_flow_catalog(dir.path(), &catalog).await.unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - let flow = catalog.get_flow("old").await.unwrap().unwrap(); - assert!( - !flow.require_approval, - "a column that did not exist is false" - ); - let run = catalog.get_flow_run("r").await.unwrap().unwrap(); - assert!(run.graph_hash.is_none()); -} - -#[tokio::test] -async fn a_row_that_does_not_decode_costs_only_itself_and_its_dependents() { - let dir = tempfile::tempdir().unwrap(); - legacy_db( - dir.path(), - &format!( - "CREATE TABLE flow_definitions (id TEXT PRIMARY KEY, name TEXT NOT NULL, graph_json TEXT NOT NULL, - enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - last_run_at TEXT, last_status TEXT, require_approval INTEGER NOT NULL DEFAULT 0); - INSERT INTO flow_definitions VALUES ('good', 'Good', '{GRAPH}', 1, 'a', 'a', NULL, NULL, 0); - INSERT INTO flow_definitions VALUES ('broken', 'Broken', 'not json', 1, 'b', 'b', NULL, NULL, 0); - CREATE TABLE flow_revisions (id TEXT PRIMARY KEY, flow_id TEXT NOT NULL, graph_json TEXT NOT NULL, - name TEXT NOT NULL, require_approval INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL); - INSERT INTO flow_revisions VALUES ('rev-ok', 'good', '{GRAPH}', 'Good', 0, 'c'); - INSERT INTO flow_revisions VALUES ('rev-orphan', 'broken', '{GRAPH}', 'Broken', 0, 'd'); - INSERT INTO flow_revisions VALUES ('rev-bad', 'good', 'nope', 'Good', 0, 'e'); - CREATE TABLE flow_runs (id TEXT PRIMARY KEY, flow_id TEXT NOT NULL, thread_id TEXT NOT NULL, status TEXT NOT NULL, - started_at TEXT NOT NULL, finished_at TEXT, steps_json TEXT NOT NULL DEFAULT '[]', - pending_approvals_json TEXT NOT NULL DEFAULT '[]', error TEXT, graph_hash TEXT); - INSERT INTO flow_runs VALUES ('run-ok', 'good', 't', 'completed', 'a', NULL, '[]', '[]', NULL, NULL); - INSERT INTO flow_runs VALUES ('run-orphan', 'broken', 't', 'completed', 'b', NULL, '[]', '[]', NULL, NULL); - INSERT INTO flow_runs VALUES ('run-bad-steps', 'good', 't', 'completed', 'c', NULL, 'oops', '[]', NULL, NULL); - INSERT INTO flow_runs VALUES ('run-bad-approvals', 'good', 't', 'completed', 'd', NULL, '[]', 'oops', NULL, NULL); - CREATE TABLE flow_state (namespace TEXT NOT NULL, key TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY (namespace, key)); - INSERT INTO flow_state VALUES ('n', 'ok', '{{\"a\":1}}'); - INSERT INTO flow_state VALUES ('n', 'bad', 'not json'); - CREATE TABLE flow_suggestions (id TEXT PRIMARY KEY, title TEXT NOT NULL, one_liner TEXT NOT NULL, rationale TEXT NOT NULL, - trigger_hint TEXT, steps_json TEXT NOT NULL DEFAULT '[]', connections_json TEXT NOT NULL DEFAULT '[]', - slugs_json TEXT NOT NULL DEFAULT '[]', build_prompt TEXT NOT NULL, confidence REAL NOT NULL DEFAULT 0, - status TEXT NOT NULL DEFAULT 'new', created_at TEXT NOT NULL, source_run_id TEXT); - INSERT INTO flow_suggestions VALUES ('s-ok', 't', 'o', 'r', NULL, '[]', '[]', '[]', 'b', 0.5, 'new', 'a', NULL); - INSERT INTO flow_suggestions VALUES ('s-bad', 't', 'o', 'r', NULL, 'oops', '[]', '[]', 'b', 0.5, 'new', 'b', NULL);" - ), - ); - let catalog = target(); - let report = import_flow_catalog(dir.path(), &catalog).await.unwrap(); - - assert_eq!(report.tally("flows").written, 1); - assert_eq!(report.tally("flows").invalid, 1); - assert_eq!(report.tally("revisions").written, 1); - assert_eq!(report.tally("revisions").invalid, 2); - assert_eq!(report.tally("runs").written, 1); - assert_eq!(report.tally("runs").invalid, 3); - assert_eq!(report.tally("state").written, 1); - assert_eq!(report.tally("state").invalid, 1); - assert_eq!(report.tally("suggestions").written, 1); - assert_eq!(report.tally("suggestions").invalid, 1); - assert!(!report.is_clean()); - assert!( - report - .problems() - .iter() - .any(|p| p.contains("flow 'broken'")) - ); - assert!(catalog.get_flow("good").await.unwrap().is_some()); - assert!(catalog.get_flow("broken").await.unwrap().is_none()); - assert!(catalog.get_flow_run("run-orphan").await.unwrap().is_none()); -} - -#[tokio::test] -async fn drafts_that_are_not_usable_are_counted_and_staging_files_are_ignored() { - let dir = tempfile::tempdir().unwrap(); - let drafts = dir.path().join("drafts"); - std::fs::create_dir_all(&drafts).unwrap(); - let draft = |id: &str| { - json!({ "id": id, "name": "d", "graph": {}, "origin": "chat", - "created_at": "2025-01-01T00:00:00Z", "updated_at": "2025-01-01T00:00:00Z" }) - .to_string() - }; - std::fs::write(drafts.join("fine.json"), draft("fine")).unwrap(); - std::fs::write(drafts.join("garbage.json"), "not json").unwrap(); - std::fs::write(drafts.join("unsafe.json"), draft("../escape")).unwrap(); - std::fs::write(drafts.join(".fine.json.tmp"), draft("tmp")).unwrap(); - std::fs::write(drafts.join(".staged.json"), draft("staged")).unwrap(); - std::fs::write(drafts.join("notes.txt"), "not a draft").unwrap(); - std::fs::create_dir_all(drafts.join("nested.json")).unwrap(); - - let catalog = target(); - let report = import_flow_catalog(dir.path(), &catalog).await.unwrap(); - - assert_eq!(report.tally("drafts").written, 1); - assert_eq!(report.tally("drafts").invalid, 2); - assert!(catalog.get_draft("fine").await.unwrap().is_some()); - assert_eq!(catalog.list_drafts().await.unwrap().len(), 1); -} - -#[tokio::test] -async fn a_flows_db_that_is_not_a_database_is_an_error_not_a_silent_skip() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("flows.db"), - b"this is not a sqlite database, only text that is long enough to have a header", - ) - .unwrap(); - assert!(import_flow_catalog(dir.path(), &target()).await.is_err()); -} - -#[tokio::test] -async fn a_database_with_none_of_the_tables_imports_nothing() { - let dir = tempfile::tempdir().unwrap(); - legacy_db(dir.path(), "CREATE TABLE unrelated (x INTEGER);"); - let report = import_flow_catalog(dir.path(), &target()).await.unwrap(); - assert_eq!(report.total_written(), 0); -} - -#[tokio::test] -async fn an_unlistable_drafts_path_is_an_error() { - let dir = tempfile::tempdir().unwrap(); - // `drafts` is a file, so it cannot be listed as a directory. - std::fs::write(dir.path().join("drafts"), "x").unwrap(); - assert!(import_flow_catalog(dir.path(), &target()).await.is_err()); -} diff --git a/crates/tinyflows-import/src/checkpoint_reader.rs b/crates/tinyflows-import/src/checkpoint_reader.rs deleted file mode 100644 index eed34892..00000000 --- a/crates/tinyflows-import/src/checkpoint_reader.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Bounded host-side readers for the retired SQLite and JSONL checkpointers. - -use std::future::Future; -use std::io::{self, BufRead, BufReader}; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use tinyflows_bus::catalog::{ - LegacyImportKind, LegacyImportRecord, LegacySqliteCheckpointWrite, - MAX_IMPORT_RECORD_PAYLOAD_BYTES, -}; - -use crate::flow_reader::BatchSender; -use crate::legacy_db::LegacyDb; - -// Read one source row at a time so a page cannot retain several maximum-sized -// payloads while waiting to be converted into bounded callback batches. -const SQLITE_PAGE_ROWS: i64 = 1; -const CHECKPOINT_WRITES_QUERY: &str = "WITH source_rows AS (SELECT rowid, thread_id, namespace, checkpoint_id, task_id, idx, node, channel, payload, COALESCE(octet_length(thread_id), 0) + COALESCE(octet_length(namespace), 0) + COALESCE(octet_length(checkpoint_id), 0) + COALESCE(octet_length(task_id), 0) + COALESCE(octet_length(node), 0) + COALESCE(octet_length(channel), 0) + COALESCE(octet_length(payload), 0) AS raw_bytes FROM checkpoint_writes) SELECT rowid, raw_bytes <= ?1, CASE WHEN raw_bytes <= ?1 THEN CAST(thread_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(namespace AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(checkpoint_id AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(task_id AS BLOB) END, idx, CASE WHEN raw_bytes <= ?1 THEN CAST(node AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(channel AS BLOB) END, CASE WHEN raw_bytes <= ?1 THEN CAST(payload AS BLOB) END FROM source_rows ORDER BY rowid LIMIT ?2 OFFSET ?3"; - -/// Reads SQLite and file/JSONL checkpoint sources as bounded callback batches. -/// -/// A torn invalid final JSONL line is ignored as the retired file reader did. -/// The source paths remain host-side and are never serialized into records. -pub async fn read_legacy_checkpoint_batches( - sqlite_path: &Path, - file_dir: &Path, - send: F, -) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - read_sources(Some(sqlite_path), Some(file_dir), send).await -} - -/// Reads only the retired SQLite checkpoint database as bounded raw batches. -pub async fn read_legacy_sqlite_checkpoint_batches( - sqlite_path: &Path, - send: F, -) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - read_sources(Some(sqlite_path), None, send).await -} - -/// Reads only the retired file/JSONL checkpoint directory as bounded raw batches. -pub async fn read_legacy_file_checkpoint_batches(file_dir: &Path, send: F) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - read_sources(None, Some(file_dir), send).await -} - -async fn read_sources( - sqlite_path: Option<&Path>, - file_dir: Option<&Path>, - mut send: F, -) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let mut batches = BatchSender::default(); - if let Some(sqlite_path) = sqlite_path - && let Some(db) = LegacyDb::open(sqlite_path)? - { - let tables = db.tables().await?; - if tables.contains("checkpoints") { - let mut offset = 0_i64; - loop { - let rows = db - .read("checkpoints", move |connection| { - let mut statement = connection.prepare( - "SELECT CASE WHEN octet_length(record) <= ?1 THEN CAST(record AS BLOB) END, seq FROM checkpoints ORDER BY seq LIMIT ?2 OFFSET ?3", - )?; - statement - .query_map( - [MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64, SQLITE_PAGE_ROWS, offset], - |row| { - Ok(( - row.get::<_, Option>>(0)?, - row.get::<_, i64>(1)?, - )) - }, - )? - .collect::, _>>() - }) - .await?; - if rows.is_empty() { - break; - } - offset += rows.len() as i64; - for (raw, sequence) in rows { - let record = match raw.and_then(|bytes| String::from_utf8(bytes).ok()) { - Some(raw_json) => LegacyImportRecord::Checkpoint { raw_json }, - None => LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Checkpoint, - identity: Some(format!("seq:{sequence}")), - message: "checkpoint source record exceeds the import byte limit or is not UTF-8".into(), - }, - }; - batches.push(record, &mut send).await?; - } - } - } - if tables.contains("checkpoint_writes") { - let mut offset = 0_i64; - loop { - let rows = db - .read("checkpoint writes", move |connection| { - let mut statement = connection.prepare(CHECKPOINT_WRITES_QUERY)?; - let mapped = statement.query_map( - [ - MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64, - SQLITE_PAGE_ROWS, - offset, - ], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, bool>(1)?, - row.get::<_, Option>>(2)?, - row.get::<_, Option>>(3)?, - row.get::<_, Option>>(4)?, - row.get::<_, Option>>(5)?, - row.get::<_, i64>(6)?, - row.get::<_, Option>>(7)?, - row.get::<_, Option>>(8)?, - row.get::<_, Option>>(9)?, - )) - }, - )?; - mapped.collect::, _>>() - }) - .await?; - if rows.is_empty() { - break; - } - offset += rows.len() as i64; - for ( - row_id, - within_limit, - thread, - namespace, - checkpoint, - task, - idx, - node, - channel, - payload, - ) in rows - { - let record = if within_limit { - let decoded = (|| { - Some(LegacySqliteCheckpointWrite { - thread_id: String::from_utf8(thread?).ok()?, - namespace_json: String::from_utf8(namespace?).ok()?, - checkpoint_id: String::from_utf8(checkpoint?).ok()?, - task_id: String::from_utf8(task?).ok()?, - idx, - node: String::from_utf8(node?).ok()?, - channel: String::from_utf8(channel?).ok()?, - payload_json: String::from_utf8(payload?).ok()?, - }) - })(); - decoded.map_or_else( - || LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Write, - identity: Some(format!("row:{row_id}")), - message: - "checkpoint write row contains a missing or non-UTF-8 field" - .into(), - }, - LegacyImportRecord::SqliteCheckpointWrite, - ) - } else { - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Write, - identity: Some(format!("row:{row_id}")), - message: "checkpoint write row exceeds the import byte limit".into(), - } - }; - batches.push(record, &mut send).await?; - } - } - } - } - if let Some(file_dir) = file_dir { - read_file_checkpoints(file_dir, &mut batches, &mut send).await?; - } - batches.flush(&mut send).await -} - -async fn read_file_checkpoints( - dir: &Path, - batches: &mut BatchSender, - send: &mut F, -) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())), - }; - let mut files: Vec = Vec::new(); - for entry in entries { - let path = entry - .with_context(|| format!("reading an entry under {}", dir.display()))? - .path(); - if path.is_file() { - files.push(path); - } - } - files.sort(); - for path in files { - let Some(name) = path.file_name().and_then(|name| name.to_str()) else { - continue; - }; - if let Some(stem) = name.strip_suffix(".writes.jsonl") { - let Some(thread_id) = unescape_thread_id(stem) else { - batches - .push( - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Write, - identity: Some(name.chars().take(128).collect()), - message: "not a valid checkpoint thread filename".into(), - }, - send, - ) - .await?; - continue; - }; - read_jsonl( - &path, - name, - thread_id, - LegacyImportKind::Write, - batches, - send, - ) - .await?; - } else if name.ends_with(".jsonl") { - let Some(thread_id) = name.strip_suffix(".jsonl").and_then(unescape_thread_id) else { - batches - .push( - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Checkpoint, - identity: Some(name.chars().take(128).collect()), - message: "not a valid checkpoint thread filename".into(), - }, - send, - ) - .await?; - continue; - }; - read_jsonl( - &path, - name, - thread_id, - LegacyImportKind::Checkpoint, - batches, - send, - ) - .await?; - } - } - Ok(()) -} - -async fn read_jsonl( - path: &Path, - name: &str, - thread_id: String, - kind: LegacyImportKind, - batches: &mut BatchSender, - send: &mut F, -) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let file = std::fs::File::open(path).with_context(|| format!("opening {}", path.display()))?; - let mut reader = BufReader::new(file); - while let Some((line, terminated, oversized)) = read_bounded_line(&mut reader)? { - if oversized { - batches - .push( - LegacyImportRecord::Invalid { - record_kind: kind, - identity: Some(thread_id.chars().take(128).collect()), - message: "checkpoint JSONL record exceeds the import byte limit".into(), - }, - send, - ) - .await?; - continue; - } - if line.iter().all(u8::is_ascii_whitespace) { - continue; - } - if !terminated && serde_json::from_slice::(&line).is_err() { - tracing::warn!(target: "tinyflows_import", file = name, "[import] ignoring a torn trailing checkpoint line"); - continue; - } - let Ok(raw_json) = String::from_utf8(line) else { - batches - .push( - LegacyImportRecord::Invalid { - record_kind: kind, - identity: Some(thread_id.chars().take(128).collect()), - message: "checkpoint JSONL record is not UTF-8".into(), - }, - send, - ) - .await?; - continue; - }; - let record = match kind { - LegacyImportKind::Checkpoint => LegacyImportRecord::Checkpoint { raw_json }, - LegacyImportKind::Write => LegacyImportRecord::FileCheckpointWrite { - thread_id: thread_id.clone(), - raw_json, - }, - _ => unreachable!("only checkpoint and write files are read here"), - }; - batches.push(record, send).await?; - } - Ok(()) -} - -fn read_bounded_line(reader: &mut R) -> io::Result, bool, bool)>> { - let mut line = Vec::new(); - let mut saw_bytes = false; - let mut oversized = false; - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - return Ok(saw_bytes.then_some((line, false, oversized))); - } - saw_bytes = true; - let newline = available.iter().position(|byte| *byte == b'\n'); - let consumed = newline.map_or(available.len(), |index| index + 1); - let content_len = newline.unwrap_or(consumed); - if !oversized { - if line.len() + content_len > MAX_IMPORT_RECORD_PAYLOAD_BYTES { - oversized = true; - line.clear(); - } else { - line.extend_from_slice(&available[..content_len]); - } - } - reader.consume(consumed); - if newline.is_some() { - return Ok(Some((line, true, oversized))); - } - } -} - -pub(crate) fn unescape_thread_id(escaped: &str) -> Option { - let bytes = escaped.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut index = 0; - while index < bytes.len() { - if bytes[index] == b'%' { - let hex = escaped.get(index + 1..index + 3)?; - out.push(u8::from_str_radix(hex, 16).ok()?); - index += 3; - } else { - out.push(bytes[index]); - index += 1; - } - } - String::from_utf8(out).ok() -} - -#[cfg(test)] -#[path = "checkpoint_reader_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/checkpoint_reader_tests.rs b/crates/tinyflows-import/src/checkpoint_reader_tests.rs deleted file mode 100644 index 78b415a7..00000000 --- a/crates/tinyflows-import/src/checkpoint_reader_tests.rs +++ /dev/null @@ -1,404 +0,0 @@ -use super::*; -use crate::test_support::fixtures; -use tinyflows_bus::catalog::{MAX_IMPORT_BATCH_BYTES, MAX_IMPORT_BATCH_RECORDS}; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -#[tokio::test] -async fn sqlite_and_jsonl_checkpoint_sources_emit_raw_ordered_rows() { - for (sqlite, files) in [ - ( - fixtures().join("checkpoints/checkpoints.db"), - fixtures().join("checkpoints/absent"), - ), - ( - fixtures().join("checkpoints/absent.db"), - fixtures().join("checkpoints/file"), - ), - ] { - let batches = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let out = std::sync::Arc::clone(&batches); - let mut send = move |batch| { - let out = std::sync::Arc::clone(&out); - async move { - out.lock().unwrap().push(batch); - Ok(()) - } - }; - read_legacy_checkpoint_batches(&sqlite, &files, &mut send) - .await - .unwrap(); - let batches = batches.lock().unwrap(); - let records: Vec<_> = batches.iter().flatten().collect(); - assert_eq!( - records - .iter() - .filter(|record| matches!(record, LegacyImportRecord::Checkpoint { .. })) - .count(), - 4 - ); - assert_eq!( - records - .iter() - .filter(|record| matches!( - record, - LegacyImportRecord::SqliteCheckpointWrite(_) - | LegacyImportRecord::FileCheckpointWrite { .. } - )) - .count(), - 3 - ); - assert!( - records - .iter() - .all(|record| !matches!(record, LegacyImportRecord::Invalid { .. })) - ); - } -} - -#[tokio::test] -async fn oversized_jsonl_line_becomes_a_bounded_invalid_record() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("thread.jsonl"); - std::fs::write(&path, vec![b'x'; MAX_IMPORT_RECORD_PAYLOAD_BYTES + 1]).unwrap(); - let batches = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let out = std::sync::Arc::clone(&batches); - let mut send = move |batch| { - let out = std::sync::Arc::clone(&out); - async move { - out.lock().unwrap().push(batch); - Ok(()) - } - }; - read_legacy_checkpoint_batches(&dir.path().join("missing.db"), dir.path(), &mut send) - .await - .unwrap(); - let batches = batches.lock().unwrap(); - assert!(matches!( - &batches[0][0], - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Checkpoint, - .. - } - )); -} - -#[tokio::test] -async fn oversized_sqlite_rows_are_counted_without_returning_their_payloads() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("checkpoints.db"); - let oversized = "x".repeat(MAX_IMPORT_RECORD_PAYLOAD_BYTES + 1); - let connection = Connection::open(&path).unwrap(); - connection - .execute_batch( - "CREATE TABLE checkpoints (seq INTEGER PRIMARY KEY, record TEXT NOT NULL); - CREATE TABLE checkpoint_writes ( - thread_id TEXT NOT NULL, - namespace TEXT NOT NULL, - checkpoint_id TEXT NOT NULL, - task_id TEXT NOT NULL, - idx INTEGER NOT NULL, - node TEXT NOT NULL, - channel TEXT NOT NULL, - payload TEXT NOT NULL - );", - ) - .unwrap(); - connection - .execute( - "INSERT INTO checkpoints (seq, record) VALUES (1, ?1), (2, ?2)", - [oversized.as_str(), "{\"checkpoint_id\":\"small\"}"], - ) - .unwrap(); - connection - .execute( - "INSERT INTO checkpoint_writes VALUES - ('thread', '[]', 'c1', 'large', 0, 'node', 'channel', ?1), - ('thread', '[]', 'c1', 'small', 0, 'node', 'channel', '1')", - [&oversized], - ) - .unwrap(); - drop(connection); - - let batches = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let output = std::sync::Arc::clone(&batches); - let mut send = move |batch| { - let output = std::sync::Arc::clone(&output); - async move { - output.lock().unwrap().push(batch); - Ok(()) - } - }; - read_legacy_sqlite_checkpoint_batches(&path, &mut send) - .await - .unwrap(); - - let batches = batches.lock().unwrap(); - let records: Vec<_> = batches.iter().flatten().collect(); - assert_eq!(records.len(), 4); - assert!(matches!( - records[0], - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Checkpoint, - .. - } - )); - assert!(matches!( - records[1], - LegacyImportRecord::Checkpoint { raw_json } if raw_json == "{\"checkpoint_id\":\"small\"}" - )); - assert!(matches!( - records[2], - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Write, - .. - } - )); - assert!(matches!( - records[3], - LegacyImportRecord::SqliteCheckpointWrite(row) if row.task_id == "small" - )); - for batch in batches.iter() { - assert!(batch.len() <= MAX_IMPORT_BATCH_RECORDS); - assert!(serde_json::to_vec(batch).unwrap().len() <= MAX_IMPORT_BATCH_BYTES); - assert!(batch.iter().all(|record| match record { - LegacyImportRecord::Invalid { - identity, message, .. - } => { - identity - .as_ref() - .is_none_or(|identity| identity.len() <= 128) - && message.len() < 256 - } - LegacyImportRecord::Checkpoint { raw_json } => { - raw_json.len() <= MAX_IMPORT_RECORD_PAYLOAD_BYTES - } - LegacyImportRecord::SqliteCheckpointWrite(row) => - serde_json::to_vec(row).unwrap().len() <= MAX_IMPORT_RECORD_PAYLOAD_BYTES, - _ => false, - })); - } -} - -#[test] -fn checkpoint_write_size_guard_avoids_loading_large_values_and_continues() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch( - "CREATE TABLE checkpoint_writes ( - thread_id TEXT, namespace TEXT, checkpoint_id TEXT, task_id TEXT, - idx INTEGER, node TEXT, channel TEXT, payload TEXT - );", - ) - .unwrap(); - connection - .execute( - "INSERT INTO checkpoint_writes VALUES - ('thread', '[]', 'c1', 'large', 0, 'node', 'channel', ?1), - ('thread', '[]', 'c1', 'small', 1, 'node', 'channel', '1')", - ["x".repeat(64 * 1024)], - ) - .unwrap(); - unsafe { - tinystoragedrivers_sqlite::rusqlite::ffi::sqlite3_limit( - connection.handle(), - tinystoragedrivers_sqlite::rusqlite::ffi::SQLITE_LIMIT_LENGTH, - 4096, - ); - } - - let read_row = |offset| { - connection - .query_row(CHECKPOINT_WRITES_QUERY, [1024_i64, 1_i64, offset], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, bool>(1)?, - row.get::<_, Option>>(2)?, - row.get::<_, Option>>(9)?, - )) - }) - .unwrap() - }; - let oversized = read_row(0); - assert_eq!(oversized.0, 1); - assert!(!oversized.1); - assert!(oversized.2.is_none()); - assert!(oversized.3.is_none()); - - let later = read_row(1); - assert_eq!(later.0, 2); - assert!(later.1); - assert_eq!(later.2.as_deref(), Some(b"thread".as_slice())); - assert_eq!(later.3.as_deref(), Some(b"1".as_slice())); -} - -#[tokio::test] -async fn jsonl_reader_splits_batches_at_record_count_and_byte_budgets() { - let dir = tempfile::tempdir().unwrap(); - let count_dir = dir.path().join("count"); - std::fs::create_dir(&count_dir).unwrap(); - let count_lines = (0..=MAX_IMPORT_BATCH_RECORDS) - .map(|index| format!("{{\"id\":{index}}}\n")) - .collect::(); - std::fs::write(count_dir.join("count.jsonl"), count_lines).unwrap(); - - let count_batches = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let count_output = std::sync::Arc::clone(&count_batches); - let mut count_send = move |batch| { - let output = std::sync::Arc::clone(&count_output); - async move { - output.lock().unwrap().push(batch); - Ok(()) - } - }; - read_legacy_file_checkpoint_batches(&count_dir, &mut count_send) - .await - .unwrap(); - { - let count_batches = count_batches.lock().unwrap(); - assert_eq!(count_batches.len(), 2); - assert_eq!(count_batches[0].len(), MAX_IMPORT_BATCH_RECORDS); - assert_eq!(count_batches[1].len(), 1); - } - - let byte_dir = dir.path().join("bytes"); - std::fs::create_dir(&byte_dir).unwrap(); - let byte_lines = (0..12) - .map(|index| { - format!( - "{{\"id\":{index},\"payload\":\"{}\"}}\n", - "x".repeat(200_000) - ) - }) - .collect::(); - std::fs::write(byte_dir.join("bytes.jsonl"), byte_lines).unwrap(); - - let byte_batches = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let byte_output = std::sync::Arc::clone(&byte_batches); - let mut byte_send = move |batch| { - let output = std::sync::Arc::clone(&byte_output); - async move { - output.lock().unwrap().push(batch); - Ok(()) - } - }; - read_legacy_file_checkpoint_batches(&byte_dir, &mut byte_send) - .await - .unwrap(); - let byte_batches = byte_batches.lock().unwrap(); - assert!(byte_batches.len() > 1); - assert_eq!(byte_batches.iter().map(Vec::len).sum::(), 12); - for batch in byte_batches.iter() { - assert!(batch.len() <= MAX_IMPORT_BATCH_RECORDS); - assert!(serde_json::to_vec(batch).unwrap().len() <= MAX_IMPORT_BATCH_BYTES); - assert!(batch.iter().all(|record| matches!( - record, - LegacyImportRecord::Checkpoint { raw_json } - if raw_json.len() <= MAX_IMPORT_RECORD_PAYLOAD_BYTES - ))); - } -} - -#[cfg(feature = "checkpoint-write")] -#[tokio::test] -async fn jsonl_control_retries_keep_the_latest_value_across_import_batches() { - use tinyflows::graph::{CheckpointConfig, Checkpointer}; - use tinyflows_drivers::DriverCheckpointer; - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; - - let dir = tempfile::tempdir().unwrap(); - let thread_id = "cross-batch-thread"; - let checkpoint_id = "cross-batch-checkpoint"; - let checkpoint = serde_json::json!({ - "thread_id": thread_id, - "checkpoint_id": checkpoint_id, - "run_id": "legacy-run", - "parent_checkpoint_id": null, - "namespace": [], - "state": {}, - "next_nodes": [], - "completed_tasks": [], - "pending_writes": [], - "interrupts": [], - "pending_activations": null, - "barrier_arrivals": [], - "metadata": {} - }); - std::fs::write( - dir.path().join(format!("{thread_id}.jsonl")), - format!("{}\n", checkpoint), - ) - .unwrap(); - - let mut writes = String::new(); - for index in 0..(MAX_IMPORT_BATCH_RECORDS - 2) { - writes.push_str( - &serde_json::json!({ - "namespace": [], - "checkpoint_id": checkpoint_id, - "write": { - "node": "node", - "task_id": format!("filler-{index}"), - "idx": 0, - "channel": "result", - "payload": index - } - }) - .to_string(), - ); - writes.push('\n'); - } - for payload in ["first", "latest"] { - writes.push_str( - &serde_json::json!({ - "namespace": [], - "checkpoint_id": checkpoint_id, - "write": { - "node": "node", - "task_id": "retry", - "idx": -1, - "channel": "resume", - "payload": payload - } - }) - .to_string(), - ); - writes.push('\n'); - } - std::fs::write(dir.path().join(format!("{thread_id}.writes.jsonl")), writes).unwrap(); - - let storage = MemoryStorage::new(); - let docs = std::sync::Arc::clone( - storage - .for_scope(&Scope::new("local").unwrap()) - .unwrap() - .documents(), - ); - let checkpointer = DriverCheckpointer::::new(docs); - let sizes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let observed = std::sync::Arc::clone(&sizes); - let target = checkpointer.clone(); - read_legacy_file_checkpoint_batches(dir.path(), move |batch| { - let observed = std::sync::Arc::clone(&observed); - let target = target.clone(); - async move { - observed.lock().unwrap().push(batch.len()); - crate::legacy_checkpoint::apply_legacy_checkpoint_batch(&batch, &target).await?; - Ok(()) - } - }) - .await - .unwrap(); - - assert_eq!(*sizes.lock().unwrap(), [MAX_IMPORT_BATCH_RECORDS, 1]); - let config = CheckpointConfig { - thread_id: thread_id.into(), - checkpoint_id: Some(checkpoint_id.into()), - namespace: vec![], - }; - let writes = checkpointer.get_writes(&config).await.unwrap(); - let resumed = writes - .iter() - .find(|write| write.task_id == "retry" && write.idx == -1) - .expect("the legacy control write is present"); - assert_eq!(resumed.payload, serde_json::json!("latest")); -} diff --git a/crates/tinyflows-import/src/checkpoints.rs b/crates/tinyflows-import/src/checkpoints.rs deleted file mode 100644 index d3c956c9..00000000 --- a/crates/tinyflows-import/src/checkpoints.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! Public checkpoint imports composed from bounded readers and the shared -//! semantic checkpoint writer. - -use std::path::Path; -use std::sync::{Arc, Mutex}; - -#[cfg(test)] -use crate::checkpoint_reader::unescape_thread_id; -use anyhow::{Context, Result}; -use serde_json::Value; -#[cfg(test)] -use tinyflows::graph::ids::NodeId; -#[cfg(test)] -use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; -use tinyflows_bus::catalog::LegacyImportRecord; -use tinyflows_drivers::DriverCheckpointer; - -use crate::report::ImportReport; - -/// Imports the legacy SQLite `checkpoints.db` into `checkpointer`. -pub async fn import_sqlite_checkpoints( - checkpoints_db: &Path, - checkpointer: &DriverCheckpointer, -) -> Result { - let report = Arc::new(Mutex::new(ImportReport::default())); - let output = Arc::clone(&report); - let mut send = move |records: Vec| { - let output = Arc::clone(&output); - async move { - let response = - crate::legacy_checkpoint::apply_legacy_checkpoint_batch(&records, checkpointer) - .await?; - output - .lock() - .map_err(|_| anyhow::anyhow!("checkpoint import report lock poisoned"))? - .merge_legacy_batch(response); - Ok(()) - } - }; - crate::checkpoint_reader::read_legacy_sqlite_checkpoint_batches(checkpoints_db, &mut send) - .await - .with_context(|| format!("importing checkpoints from {}", checkpoints_db.display()))?; - drop(send); - finish_report(report, "sqlite checkpoints") -} - -/// Imports the retired `FileCheckpointer` JSONL directory. -pub async fn import_file_checkpoints( - dir: &Path, - checkpointer: &DriverCheckpointer, -) -> Result { - let report = Arc::new(Mutex::new(ImportReport::default())); - let output = Arc::clone(&report); - let mut send = move |records: Vec| { - let output = Arc::clone(&output); - async move { - let response = - crate::legacy_checkpoint::apply_legacy_checkpoint_batch(&records, checkpointer) - .await?; - output - .lock() - .map_err(|_| anyhow::anyhow!("checkpoint import report lock poisoned"))? - .merge_legacy_batch(response); - Ok(()) - } - }; - crate::checkpoint_reader::read_legacy_file_checkpoint_batches(dir, &mut send) - .await - .with_context(|| format!("importing checkpoints from {}", dir.display()))?; - drop(send); - finish_report(report, "file checkpoints") -} - -fn finish_report(report: Arc>, source: &str) -> Result { - let report = Arc::try_unwrap(report) - .map_err(|_| anyhow::anyhow!("checkpoint import report still has active readers"))? - .into_inner() - .map_err(|_| anyhow::anyhow!("checkpoint import report lock poisoned"))?; - tracing::info!(target: "tinyflows_import", written = report.total_written(), problems = report.problems().len(), "[import] {source}"); - Ok(report) -} - -#[cfg(test)] -#[path = "checkpoints_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/checkpoints_tests.rs b/crates/tinyflows-import/src/checkpoints_tests.rs deleted file mode 100644 index e3ef4c6a..00000000 --- a/crates/tinyflows-import/src/checkpoints_tests.rs +++ /dev/null @@ -1,415 +0,0 @@ -use super::*; -use crate::test_support::{docs, expected, fixtures, json, tree_digest}; -use serde_json::json; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -fn target() -> DriverCheckpointer { - DriverCheckpointer::new(docs()) -} - -fn sqlite_db() -> std::path::PathBuf { - fixtures().join("checkpoints/checkpoints.db") -} - -fn file_dir() -> std::path::PathBuf { - fixtures().join("checkpoints/file") -} - -fn c2() -> CheckpointConfig { - CheckpointConfig { - thread_id: "run-1".into(), - checkpoint_id: Some("c2".into()), - namespace: vec![], - } -} - -/// Both legacy backends held the same data: whole threads, namespaced -/// subgraph checkpoints, a thread whose id needed escaping, and writes. -async fn assert_fixture_data_came_across(checkpointer: &DriverCheckpointer) { - let expected = expected(); - let expected = &expected["checkpoints"]; - for thread in ["run-1", "Run Two/Upper"] { - let imported = checkpointer.get_thread(thread).await.unwrap(); - assert_eq!( - Value::Array(imported.iter().map(json).collect()), - expected["threads"][thread], - "thread {thread}" - ); - } - let writes = checkpointer.get_writes(&c2()).await.unwrap(); - assert_eq!( - Value::Array(writes.iter().map(json).collect()), - expected["writes"] - ); - // The subgraph's checkpoint stays in its own namespace. - let sub = checkpointer - .get_scoped("run-1", None, &["sub".to_string()]) - .await - .unwrap() - .unwrap(); - assert_eq!(sub.checkpoint_id, "s1"); - let root = checkpointer.get("run-1", None).await.unwrap().unwrap(); - assert_eq!(root.checkpoint_id, "s1", "the latest written, as before"); - let lineage = checkpointer - .state_history("run-1", &[], None) - .await - .unwrap(); - assert_eq!( - lineage - .iter() - .map(|t| t.checkpoint.checkpoint_id.as_str()) - .collect::>(), - ["c2", "c1"] - ); -} - -#[tokio::test] -async fn sqlite_checkpoints_and_writes_come_across_in_order() { - let checkpointer = target(); - let report = import_sqlite_checkpoints(&sqlite_db(), &checkpointer) - .await - .unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - assert_eq!(report.tally("checkpoints").written, 4); - assert_eq!(report.tally("writes").written, 3); - assert_fixture_data_came_across(&checkpointer).await; -} - -#[tokio::test] -async fn file_checkpoints_and_writes_come_across_and_a_torn_last_line_is_forgiven() { - let checkpointer = target(); - let report = import_file_checkpoints(&file_dir(), &checkpointer) - .await - .unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - assert_eq!(report.tally("checkpoints").written, 4); - assert_eq!(report.tally("writes").written, 3); - assert_fixture_data_came_across(&checkpointer).await; - // The torn line is real: the file does not end in a newline. - let torn = std::fs::read_to_string(file_dir().join("run-1.jsonl")).unwrap(); - assert!(!torn.ends_with('\n')); -} - -#[tokio::test] -async fn importing_again_writes_nothing_and_keeps_each_thread_in_order() { - let checkpointer = target(); - let first = import_sqlite_checkpoints(&sqlite_db(), &checkpointer) - .await - .unwrap(); - let second = import_sqlite_checkpoints(&sqlite_db(), &checkpointer) - .await - .unwrap(); - assert_eq!(second.total_written(), 0); - assert_eq!( - second.tally("checkpoints").existing, - first.tally("checkpoints").written - ); - assert_eq!( - second.tally("writes").existing, - first.tally("writes").written - ); - assert_fixture_data_came_across(&checkpointer).await; -} - -#[tokio::test] -async fn the_second_backend_adds_nothing_when_the_first_already_brought_the_same_data() { - let checkpointer = target(); - import_sqlite_checkpoints(&sqlite_db(), &checkpointer) - .await - .unwrap(); - let from_files = import_file_checkpoints(&file_dir(), &checkpointer) - .await - .unwrap(); - assert_eq!(from_files.total_written(), 0); - assert_fixture_data_came_across(&checkpointer).await; -} - -#[tokio::test] -async fn the_legacy_files_are_left_exactly_as_they_were() { - let before = tree_digest(&fixtures()); - import_sqlite_checkpoints(&sqlite_db(), &target()) - .await - .unwrap(); - import_file_checkpoints(&file_dir(), &target()) - .await - .unwrap(); - assert_eq!(tree_digest(&fixtures()), before); -} - -#[tokio::test] -async fn missing_stores_are_empty_reports() { - let dir = tempfile::tempdir().unwrap(); - let checkpointer = target(); - assert_eq!( - import_sqlite_checkpoints(&dir.path().join("none.db"), &checkpointer) - .await - .unwrap() - .total_written(), - 0 - ); - assert_eq!( - import_file_checkpoints(&dir.path().join("none"), &checkpointer) - .await - .unwrap() - .total_written(), - 0 - ); -} - -#[tokio::test] -async fn a_database_without_the_tables_imports_nothing() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("checkpoints.db"); - Connection::open(&path) - .unwrap() - .execute_batch("CREATE TABLE thread_leases (thread_id TEXT PRIMARY KEY, owner TEXT, expires_at INTEGER); - INSERT INTO thread_leases VALUES ('t', 'o', 1);") - .unwrap(); - let report = import_sqlite_checkpoints(&path, &target()).await.unwrap(); - assert_eq!( - report.total_written(), - 0, - "leases are runtime state, not data" - ); -} - -#[tokio::test] -async fn rows_that_do_not_decode_cost_only_themselves() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("checkpoints.db"); - let good = - serde_json::to_string(&crate::checkpoints::tests::sample("t", "c1", None, &[])).unwrap(); - let conn = Connection::open(&path).unwrap(); - conn.execute_batch( - "CREATE TABLE checkpoints (seq INTEGER PRIMARY KEY AUTOINCREMENT, record TEXT NOT NULL); - CREATE TABLE checkpoint_writes (thread_id TEXT NOT NULL, namespace TEXT NOT NULL, checkpoint_id TEXT NOT NULL, - task_id TEXT NOT NULL, idx INTEGER NOT NULL, node TEXT NOT NULL, channel TEXT NOT NULL, payload TEXT NOT NULL);", - ) - .unwrap(); - conn.execute("INSERT INTO checkpoints (record) VALUES (?1)", [&good]) - .unwrap(); - conn.execute("INSERT INTO checkpoints (record) VALUES ('{broken')", []) - .unwrap(); - for (namespace, payload) in [("[]", "{\"ok\":1}"), ("not json", "1"), ("[]", "not json")] { - conn.execute( - "INSERT INTO checkpoint_writes VALUES ('t', ?1, 'c1', 'task', 0, 'n', 'ch', ?2)", - [namespace, payload], - ) - .unwrap(); - } - drop(conn); - let checkpointer = target(); - let report = import_sqlite_checkpoints(&path, &checkpointer) - .await - .unwrap(); - assert_eq!(report.tally("checkpoints").written, 1); - assert_eq!(report.tally("checkpoints").invalid, 1); - assert_eq!(report.tally("writes").written, 1); - assert_eq!(report.tally("writes").invalid, 2); -} - -mod files { - use super::*; - - fn line(thread: &str, id: &str, parent: Option<&str>) -> String { - serde_json::to_string(&sample(thread, id, parent, &[])).unwrap() - } - - #[tokio::test] - async fn a_bad_line_in_the_middle_is_counted_and_the_rest_imported() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("t.jsonl"), - format!( - "{}\nnot json\n\n{}\n", - line("t", "c1", None), - line("t", "c2", Some("c1")) - ), - ) - .unwrap(); - let checkpointer = target(); - let report = import_file_checkpoints(dir.path(), &checkpointer) - .await - .unwrap(); - assert_eq!(report.tally("checkpoints").written, 2); - assert_eq!(report.tally("checkpoints").invalid, 1); - let ids: Vec<_> = checkpointer - .get_thread("t") - .await - .unwrap() - .into_iter() - .map(|c| c.checkpoint_id) - .collect(); - assert_eq!(ids, ["c1", "c2"]); - } - - #[tokio::test] - async fn a_file_that_is_complete_but_not_a_record_is_invalid_not_torn() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("t.jsonl"), "{\"checkpoint_id\":\"x\"}\n").unwrap(); - let report = import_file_checkpoints(dir.path(), &target()) - .await - .unwrap(); - assert_eq!(report.tally("checkpoints").invalid, 1); - } - - #[tokio::test] - async fn an_existing_checkpoint_and_an_existing_control_plane_write_are_not_overwritten() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("t.jsonl"), - format!("{}\n", line("t", "c1", None)), - ) - .unwrap(); - let write = |payload: Value| { - json!({ - "namespace": [], - "checkpoint_id": "c1", - "write": { "node": "n", "task_id": "task", "idx": -1, "channel": "resume", "payload": payload }, - }) - .to_string() - }; - std::fs::write( - dir.path().join("t.writes.jsonl"), - format!("{}\n", write(json!("legacy"))), - ) - .unwrap(); - - let checkpointer = target(); - let mut live = sample("t", "c1", None, &[]); - live.state = json!({ "live": true }); - checkpointer.put(live).await.unwrap(); - checkpointer - .put_writes( - &CheckpointConfig { - thread_id: "t".into(), - checkpoint_id: Some("c1".into()), - namespace: vec![], - }, - &[PendingWrite { - node: NodeId::new("n"), - task_id: "task".into(), - idx: -1, - channel: "resume".into(), - payload: json!("live"), - }], - ) - .await - .unwrap(); - - let report = import_file_checkpoints(dir.path(), &checkpointer) - .await - .unwrap(); - - assert_eq!(report.total_written(), 0); - assert_eq!(report.tally("checkpoints").existing, 1); - assert_eq!(report.tally("writes").existing, 1); - let kept = checkpointer.get("t", Some("c1")).await.unwrap().unwrap(); - assert_eq!(kept.state, json!({ "live": true })); - let writes = checkpointer - .get_writes(&CheckpointConfig { - thread_id: "t".into(), - checkpoint_id: Some("c1".into()), - namespace: vec![], - }) - .await - .unwrap(); - assert_eq!(writes[0].payload, json!("live")); - } - - #[tokio::test] - async fn writes_sidecar_names_are_unescaped_both_ways_and_bad_ones_are_invalid() { - let dir = tempfile::tempdir().unwrap(); - let write = json!({ - "checkpoint_id": "c1", - "write": { "node": "n", "task_id": "task", "idx": 0, "channel": "ch", "payload": 1 }, - }) - .to_string(); - // Current escaping (`%52` is `R`), the older kind that left capitals alone, - // a malformed escape, and bytes that are not UTF-8. - for name in ["%52un%2F1", "Alice", "bad%ZZ", "short%4", "%FF%FE"] { - std::fs::write( - dir.path().join(format!("{name}.writes.jsonl")), - format!("{write}\n"), - ) - .unwrap(); - } - let checkpointer = target(); - let report = import_file_checkpoints(dir.path(), &checkpointer) - .await - .unwrap(); - assert_eq!(report.tally("writes").written, 2); - assert_eq!(report.tally("writes").invalid, 3); - for thread in ["Run/1", "Alice"] { - let config = CheckpointConfig { - thread_id: thread.into(), - checkpoint_id: Some("c1".into()), - namespace: vec![], - }; - assert_eq!( - checkpointer.get_writes(&config).await.unwrap().len(), - 1, - "{thread}" - ); - } - } - - #[tokio::test] - async fn things_that_are_not_checkpoint_files_are_ignored() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("README.txt"), "x").unwrap(); - std::fs::write(dir.path().join("t.jsonl.tmp.1.2"), "not a record").unwrap(); - std::fs::create_dir_all(dir.path().join("sub.jsonl")).unwrap(); - let report = import_file_checkpoints(dir.path(), &target()) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); - assert!(report.is_clean()); - } - - #[tokio::test] - async fn a_path_that_is_not_a_directory_is_an_error() { - let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("file"); - std::fs::write(&file, "x").unwrap(); - assert!(import_file_checkpoints(&file, &target()).await.is_err()); - } -} - -#[test] -fn escaping_round_trips_the_way_the_old_stores_wrote_it() { - assert_eq!( - unescape_thread_id("plain-1.x_y").as_deref(), - Some("plain-1.x_y") - ); - assert_eq!(unescape_thread_id("a%20b%2Fc").as_deref(), Some("a b/c")); - assert_eq!(unescape_thread_id("%C3%A9").as_deref(), Some("é")); - assert_eq!(unescape_thread_id("").as_deref(), Some("")); - assert!(unescape_thread_id("%").is_none()); - assert!(unescape_thread_id("%G1").is_none()); - assert!(unescape_thread_id("%C3").is_none(), "half a UTF-8 sequence"); -} - -/// A checkpoint of `thread` in `namespace`, for the files these tests write. -pub(super) fn sample( - thread: &str, - id: &str, - parent: Option<&str>, - namespace: &[&str], -) -> Checkpoint { - Checkpoint { - thread_id: thread.to_string(), - checkpoint_id: id.to_string(), - run_id: Some("run-1".to_string()), - parent_checkpoint_id: parent.map(str::to_string), - namespace: namespace.iter().map(|s| s.to_string()).collect(), - state: json!({ "id": id }), - next_nodes: vec![NodeId::new("next")], - completed_tasks: vec![NodeId::new("done")], - pending_writes: Vec::new(), - interrupts: Vec::new(), - pending_activations: None, - barrier_arrivals: Vec::new(), - metadata: json!({ "source": "loop", "step": 1 }), - } -} diff --git a/crates/tinyflows-import/src/cron.rs b/crates/tinyflows-import/src/cron.rs deleted file mode 100644 index d0784853..00000000 --- a/crates/tinyflows-import/src/cron.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! `jobs.db`: the cron store before it moved to documents, copied into a -//! `CronDocuments` store. See `legacy_cron` for the reader. - -use std::collections::HashMap; -use std::path::Path; - -use anyhow::Result; -use tinyflows_drivers::schedule::CronDocuments; - -use crate::legacy_cron::read_legacy_cron; -use crate::report::ImportReport; - -/// The jobs and run history in `jobs_db` (the file the old store was opened -/// with), copied into `cron`. -/// -/// A flow-schedule job is filed under the id the document store derives from -/// its flow, and its runs go with it. Jobs are imported with their next run, -/// last run and schedule exactly as stored; a job that is due is simply due. -/// -/// # Errors -/// -/// When the file exists but cannot be read, or the target store fails (for -/// example an active-job cap the jobs would exceed). A row that cannot be -/// decoded is counted in the report as invalid. -pub async fn import_cron(jobs_db: &Path, cron: &CronDocuments) -> Result { - let legacy = read_legacy_cron(jobs_db).await?; - let mut report = legacy.report; - let mut filed_as = HashMap::new(); - for job in &legacy.jobs { - let imported = cron.import_job(job).await?; - report.wrote("jobs", imported.written); - filed_as.insert(job.id.clone(), imported.id); - } - for mut run in legacy.runs { - let Some(id) = filed_as.get(&run.job_id) else { - report.invalid( - "runs", - format!("run {}: job '{}' was not imported", run.id, run.job_id), - ); - continue; - }; - run.job_id = id.clone(); - report.wrote("runs", cron.import_run(&run).await?); - } - tracing::info!( - target: "tinyflows_import", - written = report.total_written(), - problems = report.problems().len(), - "[import] cron jobs" - ); - Ok(report) -} - -#[cfg(test)] -#[path = "cron_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/cron_tests.rs b/crates/tinyflows-import/src/cron_tests.rs deleted file mode 100644 index a7c7decd..00000000 --- a/crates/tinyflows-import/src/cron_tests.rs +++ /dev/null @@ -1,240 +0,0 @@ -use super::*; -use crate::test_support::{docs, expected, fixtures, json, tree_digest}; -use serde_json::Value; -use tinyflows_bus::schedule::types::JobType; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -fn jobs_db() -> std::path::PathBuf { - fixtures().join("cron/jobs.db") -} - -fn target() -> CronDocuments { - CronDocuments::new(docs()) -} - -/// `value` without its `id` / `job_id`, which a flow-schedule job changes. -fn without_ids(mut value: Value) -> Value { - if let Some(object) = value.as_object_mut() { - object.remove("id"); - object.remove("job_id"); - } - value -} - -#[tokio::test] -async fn every_job_and_run_comes_across_with_its_fields() { - let cron = target(); - let report = import_cron(&jobs_db(), &cron).await.unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - assert_eq!(report.tally("jobs").written, 3); - assert_eq!(report.tally("runs").written, 4); - - let expected = expected(); - let expected = &expected["cron"]; - for legacy in expected["jobs"].as_array().unwrap() { - let legacy_id = legacy["id"].as_str().unwrap(); - let is_flow = legacy["job_type"] == "flow"; - let new_id = if is_flow { - format!("flow:{}", legacy["command"].as_str().unwrap()) - } else { - legacy_id.to_string() - }; - let imported = cron.get_job(&new_id).await.unwrap(); - assert_eq!( - without_ids(json(&imported)), - without_ids(legacy.clone()), - "job {legacy_id}" - ); - if !is_flow { - assert_eq!(imported.id, legacy_id, "an ordinary job keeps its id"); - } - let runs = cron.list_runs(&new_id, 100).await.unwrap(); - let legacy_runs: Vec = expected["runs"][legacy_id].as_array().unwrap().clone(); - assert_eq!( - runs.iter() - .map(|r| without_ids(json(r))) - .collect::>(), - legacy_runs.into_iter().map(without_ids).collect::>(), - "runs of {legacy_id}" - ); - for run in &runs { - assert_eq!(run.job_id, new_id, "runs follow their job"); - } - } -} - -#[tokio::test] -async fn run_numbers_are_kept_and_new_runs_come_after_them() { - let cron = target(); - import_cron(&jobs_db(), &cron).await.unwrap(); - let expected = expected(); - let highest = expected["cron"]["runs"] - .as_object() - .unwrap() - .values() - .flat_map(|runs| runs.as_array().unwrap()) - .map(|run| run["id"].as_i64().unwrap()) - .max() - .unwrap(); - let job = cron.list_jobs().await.unwrap().remove(0); - let now = chrono::Utc::now(); - cron.record_run(&job.id, now, now, "ok", None, 1) - .await - .unwrap(); - let newest = cron.list_runs(&job.id, 1).await.unwrap(); - assert_eq!(newest[0].id, highest + 1); -} - -#[tokio::test] -async fn a_flow_schedule_job_is_found_by_its_flow_after_the_import() { - let cron = target(); - import_cron(&jobs_db(), &cron).await.unwrap(); - let found = cron - .find_flow_schedule_job("flow-beta") - .await - .unwrap() - .unwrap(); - assert_eq!(found.command, "flow-beta"); - assert_eq!(found.id, "flow:flow-beta"); - // Registering the flow again returns it rather than adding a second. - let again = cron - .add_flow_schedule_job( - "flow-beta", - tinyflows_schedule::Schedule::Every { every_ms: 5_000 }, - ) - .await - .unwrap(); - assert_eq!(again.id, found.id); - assert_eq!(cron.list_jobs().await.unwrap().len(), 3); -} - -#[tokio::test] -async fn importing_again_writes_nothing() { - let cron = target(); - import_cron(&jobs_db(), &cron).await.unwrap(); - let second = import_cron(&jobs_db(), &cron).await.unwrap(); - assert_eq!(second.total_written(), 0); - assert_eq!(second.tally("jobs").existing, 3); - assert_eq!(second.tally("runs").existing, 4); - assert_eq!(cron.list_jobs().await.unwrap().len(), 3); -} - -#[tokio::test] -async fn the_legacy_files_are_left_exactly_as_they_were() { - let before = tree_digest(&fixtures()); - import_cron(&jobs_db(), &target()).await.unwrap(); - assert_eq!(tree_digest(&fixtures()), before); -} - -#[tokio::test] -async fn a_missing_file_is_an_empty_report() { - let dir = tempfile::tempdir().unwrap(); - let report = import_cron(&dir.path().join("jobs.db"), &target()) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); -} - -#[tokio::test] -async fn a_database_without_the_jobs_table_imports_nothing() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("jobs.db"); - Connection::open(&path) - .unwrap() - .execute_batch("CREATE TABLE other (x INTEGER);") - .unwrap(); - assert_eq!( - import_cron(&path, &target()).await.unwrap().total_written(), - 0 - ); -} - -fn legacy(dir: &std::path::Path, sql: &str) -> std::path::PathBuf { - let path = dir.join("jobs.db"); - Connection::open(&path).unwrap().execute_batch(sql).unwrap(); - path -} - -#[tokio::test] -async fn an_early_schema_with_only_an_expression_and_a_side_policy_table_reads_with_defaults() { - let dir = tempfile::tempdir().unwrap(); - let path = legacy( - dir.path(), - "CREATE TABLE cron_jobs (id TEXT PRIMARY KEY, expression TEXT NOT NULL, command TEXT NOT NULL, - created_at TEXT NOT NULL, next_run TEXT NOT NULL, last_run TEXT, last_status TEXT, last_output TEXT); - INSERT INTO cron_jobs VALUES ('old', '0 9 * * *', 'echo old', '2024-01-01T00:00:00Z', '2024-01-02T09:00:00Z', NULL, NULL, NULL); - CREATE TABLE cron_job_policies (job_id TEXT PRIMARY KEY, retries INTEGER, single_flight INTEGER NOT NULL); - INSERT INTO cron_job_policies VALUES ('old', 3, 1); - CREATE TABLE cron_runs (id INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL, started_at TEXT NOT NULL, - finished_at TEXT NOT NULL, status TEXT NOT NULL, output TEXT, duration_ms INTEGER); - INSERT INTO cron_runs VALUES (7, 'old', '2024-01-01T09:00:00Z', '2024-01-01T09:00:02Z', 'ok', NULL, 2000);", - ); - let cron = target(); - let report = import_cron(&path, &cron).await.unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - let job = cron.get_job("old").await.unwrap(); - assert_eq!( - job.schedule, - tinyflows_schedule::Schedule::Cron { - expr: "0 9 * * *".into(), - tz: None, - active_hours: None - } - ); - assert!(job.enabled); - assert_eq!(job.job_type, JobType::Shell); - assert_eq!(job.policy.retries, Some(3)); - assert!(job.policy.single_flight); - assert_eq!(job.claim_generation, 0); - let runs = cron.list_runs("old", 10).await.unwrap(); - assert_eq!(runs[0].id, 7); - assert!(runs[0].delivery_status.is_none()); -} - -#[tokio::test] -async fn rows_that_do_not_decode_cost_only_themselves() { - let dir = tempfile::tempdir().unwrap(); - let path = legacy( - dir.path(), - "CREATE TABLE cron_jobs (id TEXT PRIMARY KEY, expression TEXT NOT NULL, command TEXT NOT NULL, schedule TEXT, - job_type TEXT NOT NULL DEFAULT 'shell', created_at TEXT NOT NULL, next_run TEXT NOT NULL, - last_run TEXT, last_status TEXT, last_output TEXT); - INSERT INTO cron_jobs VALUES ('good', '0 9 * * *', 'echo', NULL, 'shell', '2024-01-01T00:00:00Z', '2024-01-02T09:00:00Z', NULL, NULL, NULL); - INSERT INTO cron_jobs VALUES ('no-schedule', '', 'echo', NULL, 'shell', '2024-01-01T00:00:00Z', '2024-01-02T09:00:00Z', NULL, NULL, NULL); - INSERT INTO cron_jobs VALUES ('bad-schedule', '', 'echo', '{not json', 'shell', '2024-01-01T00:00:00Z', '2024-01-02T09:00:00Z', NULL, NULL, NULL); - INSERT INTO cron_jobs VALUES ('bad-time', '0 9 * * *', 'echo', NULL, 'shell', 'yesterday', '2024-01-02T09:00:00Z', NULL, NULL, NULL); - CREATE TABLE cron_runs (id INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL, started_at TEXT NOT NULL, - finished_at TEXT NOT NULL, status TEXT NOT NULL, output TEXT, duration_ms INTEGER); - INSERT INTO cron_runs VALUES (1, 'good', '2024-01-01T09:00:00Z', '2024-01-01T09:00:02Z', 'ok', NULL, 2000); - INSERT INTO cron_runs VALUES (2, 'good', 'when', '2024-01-01T09:00:02Z', 'ok', NULL, 2000); - INSERT INTO cron_runs VALUES (3, 'bad-time', '2024-01-01T09:00:00Z', '2024-01-01T09:00:02Z', 'ok', NULL, 2000);", - ); - let cron = target(); - let report = import_cron(&path, &cron).await.unwrap(); - - assert_eq!(report.tally("jobs").written, 1); - assert_eq!(report.tally("jobs").invalid, 3); - assert_eq!(report.tally("runs").written, 1); - assert_eq!(report.tally("runs").invalid, 2, "{:?}", report.problems()); - assert_eq!(cron.list_jobs().await.unwrap().len(), 1); -} - -#[tokio::test] -async fn a_file_that_is_not_a_database_is_an_error() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("jobs.db"); - std::fs::write( - &path, - b"not a sqlite database, just a line of text that is long enough", - ) - .unwrap(); - assert!(import_cron(&path, &target()).await.is_err()); -} - -#[tokio::test] -async fn a_job_over_the_targets_active_cap_stops_the_import_with_an_error() { - let cron = target(); - cron.set_max_active_jobs(Some(1)).await.unwrap(); - let error = import_cron(&jobs_db(), &cron).await.unwrap_err(); - assert!(format!("{error:#}").contains("active job limit exceeded")); -} diff --git a/crates/tinyflows-import/src/flow_reader.rs b/crates/tinyflows-import/src/flow_reader.rs deleted file mode 100644 index f774af6c..00000000 --- a/crates/tinyflows-import/src/flow_reader.rs +++ /dev/null @@ -1,502 +0,0 @@ -//! Bounded, host-side readers for the retired flow catalog and draft files. - -use std::future::Future; -use std::io::{self, Read, Write}; -use std::path::Path; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use tinyflows_bus::catalog::{ - LegacyDraftRecord, LegacyFlowRecord, LegacyImportKind, LegacyImportRecord, - LegacyRevisionRecord, LegacyRunRecord, LegacyStateRecord, LegacySuggestionRecord, - MAX_IMPORT_BATCH_RECORDS, MAX_IMPORT_RECORD_PAYLOAD_BYTES, -}; - -use crate::legacy_db::{LegacyDb, column_or}; - -fn fields(names: &[&str]) -> Vec { - names.iter().map(|name| (*name).to_string()).collect() -} - -fn bounded_page_query(table: &str, fields: &[String], order: &str) -> String { - let size = fields - .iter() - .map(|expression| { - let expression = expression - .split_once(" AS ") - .map_or(expression.as_str(), |(expression, _)| expression); - format!("COALESCE(octet_length(({expression})), 0)") - }) - .collect::>() - .join(" + "); - let guarded = fields - .iter() - .map(|expression| { - let expression = expression - .split_once(" AS ") - .map_or(expression.as_str(), |(expression, _)| expression); - format!("CASE WHEN ({size}) <= ?1 THEN ({expression}) END") - }) - .collect::>() - .join(", "); - // An over-limit order key belongs to an invalid row; treating that key as - // NULL may reorder invalid rows, while valid rows keep the legacy ordering. - let order = order - .split(',') - .map(|key| format!("CASE WHEN COALESCE(octet_length({key}), 0) <= ?1 THEN {key} END")) - .collect::>() - .join(", "); - format!("SELECT {guarded}, (({size}) > ?1) FROM {table} ORDER BY {order} LIMIT 1 OFFSET ?2") -} - -/// Reads legacy flow database rows and draft files as bounded ordered batches. -/// -/// The caller owns source discovery, the target storage scope and migration -/// marker. Only file contents and safe file names cross the callback boundary. -/// Each callback completes before the next batch is read and sent. -pub async fn read_legacy_flow_catalog(dir: &Path, mut send: F) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let mut batches = BatchSender::default(); - if let Some(db) = LegacyDb::open(&dir.join("flows.db"))? { - let tables = db.tables().await?; - if tables.contains("flow_definitions") { - let columns = db.columns("flow_definitions").await?; - let mut fields = fields(&[ - "id", - "name", - "graph_json", - "enabled", - "created_at", - "updated_at", - "last_run_at", - "last_status", - ]); - fields.push(column_or(&columns, "require_approval", "0")); - let query = bounded_page_query("flow_definitions", &fields, "created_at,id"); - page_rows( - &db, - PageSpec { - what: "flow definitions", - query: &query, - field_count: fields.len(), - kind: LegacyImportKind::Flow, - }, - |row| { - Ok(LegacyFlowRecord { - id: row.get(0)?, - name: row.get(1)?, - graph_json: row.get(2)?, - enabled: row.get::<_, i64>(3)? != 0, - created_at: row.get(4)?, - updated_at: row.get(5)?, - last_run_at: row.get(6)?, - last_status: row.get(7)?, - require_approval: row.get::<_, i64>(8)? != 0, - }) - }, - LegacyImportRecord::Flow, - &mut batches, - &mut send, - ) - .await?; - } - if tables.contains("flow_revisions") { - let fields = fields(&[ - "id", - "flow_id", - "graph_json", - "name", - "require_approval", - "created_at", - ]); - let query = bounded_page_query("flow_revisions", &fields, "created_at,id"); - page_rows( - &db, - PageSpec { - what: "flow revisions", - query: &query, - field_count: fields.len(), - kind: LegacyImportKind::Revision, - }, - |row| { - Ok(LegacyRevisionRecord { - id: row.get(0)?, - flow_id: row.get(1)?, - graph_json: row.get(2)?, - name: row.get(3)?, - require_approval: row.get::<_, i64>(4)? != 0, - created_at: row.get(5)?, - }) - }, - LegacyImportRecord::Revision, - &mut batches, - &mut send, - ) - .await?; - } - if tables.contains("flow_runs") { - let columns = db.columns("flow_runs").await?; - let mut fields = fields(&[ - "id", - "flow_id", - "thread_id", - "status", - "started_at", - "finished_at", - "steps_json", - "pending_approvals_json", - "error", - ]); - fields.push(column_or(&columns, "graph_hash", "NULL")); - let query = bounded_page_query("flow_runs", &fields, "started_at,id"); - page_rows( - &db, - PageSpec { - what: "flow runs", - query: &query, - field_count: fields.len(), - kind: LegacyImportKind::Run, - }, - |row| { - Ok(LegacyRunRecord { - id: row.get(0)?, - flow_id: row.get(1)?, - thread_id: row.get(2)?, - status: row.get(3)?, - started_at: row.get(4)?, - finished_at: row.get(5)?, - steps_json: row.get(6)?, - pending_approvals_json: row.get(7)?, - error: row.get(8)?, - graph_hash: row.get(9)?, - }) - }, - LegacyImportRecord::Run, - &mut batches, - &mut send, - ) - .await?; - } - if tables.contains("flow_state") { - let fields = fields(&["namespace", "key", "value"]); - let query = bounded_page_query("flow_state", &fields, "namespace,key"); - page_rows( - &db, - PageSpec { - what: "flow state", - query: &query, - field_count: fields.len(), - kind: LegacyImportKind::State, - }, - |row| { - Ok(LegacyStateRecord { - namespace: row.get(0)?, - key: row.get(1)?, - value_json: row.get(2)?, - }) - }, - LegacyImportRecord::State, - &mut batches, - &mut send, - ) - .await?; - } - if tables.contains("flow_suggestions") { - let fields = fields(&[ - "id", - "title", - "one_liner", - "rationale", - "trigger_hint", - "steps_json", - "connections_json", - "slugs_json", - "build_prompt", - "confidence", - "status", - "created_at", - "source_run_id", - ]); - let query = bounded_page_query("flow_suggestions", &fields, "created_at,id"); - page_rows( - &db, - PageSpec { - what: "flow suggestions", - query: &query, - field_count: fields.len(), - kind: LegacyImportKind::Suggestion, - }, - |row| { - Ok(LegacySuggestionRecord { - id: row.get(0)?, - title: row.get(1)?, - one_liner: row.get(2)?, - rationale: row.get(3)?, - trigger_hint: row.get(4)?, - steps_json: row.get(5)?, - connections_json: row.get(6)?, - slugs_json: row.get(7)?, - build_prompt: row.get(8)?, - confidence: row.get(9)?, - status: row.get(10)?, - created_at: row.get(11)?, - source_run_id: row.get(12)?, - }) - }, - LegacyImportRecord::Suggestion, - &mut batches, - &mut send, - ) - .await?; - } - } - read_drafts(&dir.join("drafts"), &mut batches, &mut send).await?; - batches.flush(&mut send).await -} - -enum BoundedRow { - Valid(T), - Oversized, -} - -struct PageSpec<'a> { - what: &'static str, - query: &'a str, - field_count: usize, - kind: LegacyImportKind, -} - -async fn page_rows( - db: &LegacyDb, - spec: PageSpec<'_>, - decode: impl Fn( - &tinystoragedrivers_sqlite::rusqlite::Row<'_>, - ) -> tinystoragedrivers_sqlite::rusqlite::Result - + Send - + Sync - + 'static, - to_record: ToRecord, - batches: &mut BatchSender, - send: &mut F, -) -> Result<()> -where - T: Send + 'static, - ToRecord: Fn(T) -> LegacyImportRecord, - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let PageSpec { - what, - query, - field_count, - kind, - } = spec; - let query = query.to_string(); - let decode = Arc::new(decode); - let mut offset = 0_i64; - loop { - let statement_query = query.clone(); - let decode = Arc::clone(&decode); - let rows = db - .read(what, move |conn| { - let mut statement = conn.prepare(&statement_query)?; - let mapped = statement.query_map( - [MAX_IMPORT_RECORD_PAYLOAD_BYTES as i64, offset], - |row| { - if row.get::<_, bool>(field_count)? { - Ok(BoundedRow::Oversized) - } else { - decode(row).map(BoundedRow::Valid) - } - }, - )?; - mapped.collect::, _>>() - }) - .await?; - if rows.is_empty() { - break; - } - offset += rows.len() as i64; - for row in rows { - let record = match row { - BoundedRow::Valid(row) => to_record(row), - BoundedRow::Oversized => LegacyImportRecord::Invalid { - record_kind: kind, - identity: None, - message: "legacy source row exceeds the import byte limit".into(), - }, - }; - batches.push(record, send).await?; - } - } - Ok(()) -} - -async fn read_drafts(dir: &Path, batches: &mut BatchSender, send: &mut F) -> Result<()> -where - F: FnMut(Vec) -> Fut, - Fut: Future>, -{ - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())), - }; - let mut paths: Vec<_> = entries - .filter_map(|entry| entry.ok().map(|entry| entry.path())) - .filter(|path| { - path.is_file() - && path.extension().is_some_and(|ext| ext == "json") - && !path - .file_name() - .is_some_and(|name| name.to_string_lossy().starts_with('.')) - }) - .collect(); - paths.sort(); - for path in paths { - let file_name = path - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(); - let file = - std::fs::File::open(&path).with_context(|| format!("opening {}", path.display()))?; - let bytes = read_bounded(file, MAX_IMPORT_RECORD_PAYLOAD_BYTES) - .with_context(|| format!("reading {}", path.display()))?; - let record = if bytes.len() > MAX_IMPORT_RECORD_PAYLOAD_BYTES { - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Draft, - identity: Some(file_name.chars().take(128).collect()), - message: "draft file exceeds the import byte limit".into(), - } - } else { - LegacyImportRecord::Draft(LegacyDraftRecord { file_name, bytes }) - }; - batches.push(record, send).await?; - } - Ok(()) -} - -fn read_bounded(mut reader: impl Read, limit: usize) -> io::Result> { - let mut bytes = Vec::new(); - reader - .by_ref() - .take(limit.saturating_add(1) as u64) - .read_to_end(&mut bytes)?; - Ok(bytes) -} - -#[derive(Default)] -pub(crate) struct BatchSender { - current: Vec, - bytes: usize, -} - -impl BatchSender { - pub(crate) async fn push( - &mut self, - mut record: LegacyImportRecord, - send: &mut F, - ) -> Result<()> - where - F: FnMut(Vec) -> Fut, - Fut: Future>, - { - let mut size = serialized_size(&record, MAX_IMPORT_RECORD_PAYLOAD_BYTES)?; - if size.is_none() { - let kind = record.record_kind(); - let identity = record_identity(&record); - record = LegacyImportRecord::Invalid { - record_kind: kind, - identity, - message: "record exceeds the import byte limit".into(), - }; - size = serialized_size(&record, MAX_IMPORT_RECORD_PAYLOAD_BYTES)?; - } - let size = size.context("bounded invalid import record exceeds its byte limit")?; - if !self.current.is_empty() - && (self.current.len() >= MAX_IMPORT_BATCH_RECORDS - || self.bytes + size > MAX_IMPORT_RECORD_PAYLOAD_BYTES) - { - self.flush(send).await?; - } - self.bytes += size; - self.current.push(record); - Ok(()) - } - - pub(crate) async fn flush(&mut self, send: &mut F) -> Result<()> - where - F: FnMut(Vec) -> Fut, - Fut: Future>, - { - if self.current.is_empty() { - return Ok(()); - } - self.bytes = 0; - send(std::mem::take(&mut self.current)).await - } -} - -#[derive(Default)] -struct SizeCounter { - bytes: usize, - limit: usize, - exceeded: bool, -} - -impl Write for SizeCounter { - fn write(&mut self, bytes: &[u8]) -> io::Result { - let Some(next) = self.bytes.checked_add(bytes.len()) else { - self.exceeded = true; - return Err(io::Error::other( - "serialized import record exceeds its byte limit", - )); - }; - if next > self.limit { - self.exceeded = true; - return Err(io::Error::other( - "serialized import record exceeds its byte limit", - )); - } - self.bytes = next; - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -fn serialized_size(value: &T, limit: usize) -> Result> { - let mut counter = SizeCounter { - limit, - ..SizeCounter::default() - }; - match serde_json::to_writer(&mut counter, value) { - Ok(()) => Ok(Some(counter.bytes)), - Err(_) if counter.exceeded => Ok(None), - Err(error) => Err(error.into()), - } -} - -fn record_identity(record: &LegacyImportRecord) -> Option { - let value = match record { - LegacyImportRecord::Flow(row) => Some(row.id.as_str()), - LegacyImportRecord::Revision(row) => Some(row.id.as_str()), - LegacyImportRecord::Run(row) => Some(row.id.as_str()), - LegacyImportRecord::State(row) => Some(row.key.as_str()), - LegacyImportRecord::Suggestion(row) => Some(row.id.as_str()), - LegacyImportRecord::Draft(row) => Some(row.file_name.as_str()), - LegacyImportRecord::FileCheckpointWrite { thread_id, .. } => Some(thread_id.as_str()), - LegacyImportRecord::SqliteCheckpointWrite(row) => Some(row.task_id.as_str()), - LegacyImportRecord::Checkpoint { .. } | LegacyImportRecord::Invalid { .. } => None, - }; - value.map(|identity| identity.chars().take(128).collect()) -} - -#[cfg(test)] -#[path = "flow_reader_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/flow_reader_tests.rs b/crates/tinyflows-import/src/flow_reader_tests.rs deleted file mode 100644 index 204ab57a..00000000 --- a/crates/tinyflows-import/src/flow_reader_tests.rs +++ /dev/null @@ -1,348 +0,0 @@ -use super::*; -use std::io::Cursor; -use std::sync::Mutex; -use tinyflows_bus::catalog::LegacyImportKind; -use tinyflows_bus::catalog::{ - CallbackAddress, CatalogStorage, ImportLegacyBatchRequest, MAX_IMPORT_BATCH_BYTES, - MAX_IMPORT_RECORD_PAYLOAD_BYTES, MAX_IMPORT_REQUEST_ENVELOPE_BYTES, -}; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -#[tokio::test] -async fn batches_are_bounded_and_a_single_oversized_record_becomes_invalid() { - let mut sender = BatchSender::default(); - let batches = Arc::new(Mutex::new(Vec::new())); - let captured = Arc::clone(&batches); - let mut send = move |batch| { - let captured = Arc::clone(&captured); - async move { - captured.lock().unwrap().push(batch); - Ok(()) - } - }; - for index in 0..=MAX_IMPORT_BATCH_RECORDS { - sender - .push( - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Flow, - identity: Some(index.to_string()), - message: "source problem".into(), - }, - &mut send, - ) - .await - .unwrap(); - } - sender.flush(&mut send).await.unwrap(); - { - let batches = batches.lock().unwrap(); - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].len(), MAX_IMPORT_BATCH_RECORDS); - assert_eq!(batches[1].len(), 1); - assert!( - batches - .iter() - .all(|batch| batch.len() <= MAX_IMPORT_BATCH_RECORDS) - ); - } - let mut sender = BatchSender::default(); - let oversized = LegacyImportRecord::Flow(LegacyFlowRecord { - id: "i".repeat(10_000), - name: "legacy".into(), - graph_json: "x".repeat(MAX_IMPORT_BATCH_BYTES + 1), - enabled: true, - created_at: "2025-01-01T00:00:00Z".into(), - updated_at: "2025-01-01T00:00:00Z".into(), - last_run_at: None, - last_status: None, - require_approval: false, - }); - let captured = Arc::new(Mutex::new(Vec::new())); - let out = Arc::clone(&captured); - let mut send = move |batch| { - let out = Arc::clone(&out); - async move { - out.lock().unwrap().push(batch); - Ok(()) - } - }; - sender.push(oversized, &mut send).await.unwrap(); - sender.flush(&mut send).await.unwrap(); - let batches = captured.lock().unwrap(); - let LegacyImportRecord::Invalid { - record_kind, - identity, - .. - } = &batches[0][0] - else { - panic!("oversized record should be represented as invalid") - }; - assert_eq!(*record_kind, LegacyImportKind::Flow); - assert_eq!(identity.as_ref().unwrap().len(), 128); -} - -#[tokio::test] -async fn escape_expansion_is_bounded_and_later_records_still_import() { - let mut sender = BatchSender::default(); - let escaped = LegacyImportRecord::Flow(LegacyFlowRecord { - id: "escaped".into(), - name: "legacy".into(), - graph_json: "\u{1}".repeat(MAX_IMPORT_RECORD_PAYLOAD_BYTES / 4), - enabled: true, - created_at: "2025-01-01T00:00:00Z".into(), - updated_at: "2025-01-01T00:00:00Z".into(), - last_run_at: None, - last_status: None, - require_approval: false, - }); - let valid = LegacyImportRecord::Flow(LegacyFlowRecord { - id: "later".into(), - name: "valid".into(), - graph_json: "{}".into(), - enabled: true, - created_at: "2025-01-02T00:00:00Z".into(), - updated_at: "2025-01-02T00:00:00Z".into(), - last_run_at: None, - last_status: None, - require_approval: false, - }); - let batches = Arc::new(Mutex::new(Vec::new())); - let output = Arc::clone(&batches); - let mut send = move |batch| { - let output = Arc::clone(&output); - async move { - output.lock().unwrap().push(batch); - Ok(()) - } - }; - sender.push(escaped, &mut send).await.unwrap(); - sender.push(valid, &mut send).await.unwrap(); - sender.flush(&mut send).await.unwrap(); - - let batches = batches.lock().unwrap(); - assert!(matches!( - batches[0][0], - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Flow, - .. - } - )); - assert!(matches!( - batches[0][1], - LegacyImportRecord::Flow(ref row) if row.id == "later" - )); -} - -#[test] -fn sqlite_octet_length_counts_text_bytes() { - let connection = Connection::open_in_memory().unwrap(); - let bytes = connection - .query_row("SELECT octet_length('é')", [], |row| row.get::<_, i64>(0)) - .unwrap(); - assert_eq!(bytes, 2); -} - -#[tokio::test] -async fn oversized_catalog_rows_and_drafts_are_invalid_but_later_records_survive() { - let dir = tempfile::tempdir().unwrap(); - let db_path = dir.path().join("flows.db"); - let oversized_graph = "x".repeat(MAX_IMPORT_RECORD_PAYLOAD_BYTES + 1); - let connection = Connection::open(&db_path).unwrap(); - connection - .execute_batch( - "CREATE TABLE flow_definitions ( - id TEXT, name TEXT, graph_json TEXT, enabled INTEGER, - created_at TEXT, updated_at TEXT, last_run_at TEXT, - last_status TEXT, require_approval INTEGER - );", - ) - .unwrap(); - connection - .execute( - "INSERT INTO flow_definitions VALUES - ('oversized', 'large graph', ?1, 1, '2025-01-01', '2025-01-01', NULL, NULL, 0), - ('later-flow', 'valid graph', '{}', 1, '2025-01-02', '2025-01-02', NULL, NULL, 0)", - [&oversized_graph], - ) - .unwrap(); - drop(connection); - - let drafts = dir.path().join("drafts"); - std::fs::create_dir(&drafts).unwrap(); - std::fs::write( - drafts.join("a-oversized.json"), - vec![b'x'; MAX_IMPORT_RECORD_PAYLOAD_BYTES + 1], - ) - .unwrap(); - std::fs::write(drafts.join("z-valid.json"), br#"{"draft":true}"#).unwrap(); - - let output = Arc::new(Mutex::new(Vec::new())); - let captured = Arc::clone(&output); - read_legacy_flow_catalog(dir.path(), move |batch| { - let captured = Arc::clone(&captured); - async move { - captured.lock().unwrap().extend(batch); - Ok(()) - } - }) - .await - .unwrap(); - - let output = output.lock().unwrap(); - assert!(output.iter().any(|record| matches!( - record, - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Flow, - .. - } - ))); - assert!(output.iter().any(|record| matches!( - record, - LegacyImportRecord::Flow(row) if row.id == "later-flow" - ))); - assert!(output.iter().any(|record| matches!( - record, - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Draft, - identity: Some(name), - .. - } if name == "a-oversized.json" - ))); - assert!(output.iter().any(|record| matches!( - record, - LegacyImportRecord::Draft(row) if row.file_name == "z-valid.json" - ))); -} - -#[test] -fn oversized_sqlite_projection_avoids_materializing_the_source_value() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch("CREATE TABLE legacy_rows (id TEXT, body TEXT);") - .unwrap(); - connection - .execute( - "INSERT INTO legacy_rows VALUES ('large', ?1)", - ["x".repeat(256 * 1024)], - ) - .unwrap(); - unsafe { - tinystoragedrivers_sqlite::rusqlite::ffi::sqlite3_limit( - connection.handle(), - tinystoragedrivers_sqlite::rusqlite::ffi::SQLITE_LIMIT_LENGTH, - 4096, - ); - } - - let query = bounded_page_query("legacy_rows", &fields(&["id", "body"]), "id"); - let row = connection - .query_row(&query, [1024_i64, 0_i64], |row| { - Ok(( - row.get::<_, Option>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, bool>(2)?, - )) - }) - .unwrap(); - assert_eq!(row, (None, None, true)); -} - -#[test] -fn oversized_sqlite_order_key_is_guarded_and_valid_rows_keep_reading() { - let connection = Connection::open_in_memory().unwrap(); - connection - .execute_batch("CREATE TABLE legacy_rows (id TEXT, created_at TEXT, body TEXT);") - .unwrap(); - connection - .execute( - "INSERT INTO legacy_rows VALUES - ('oversized', ?1, '{}'), - ('later', '2025-01-01', '{}')", - ["x".repeat(64 * 1024)], - ) - .unwrap(); - unsafe { - tinystoragedrivers_sqlite::rusqlite::ffi::sqlite3_limit( - connection.handle(), - tinystoragedrivers_sqlite::rusqlite::ffi::SQLITE_LIMIT_LENGTH, - 4096, - ); - } - - let query = bounded_page_query( - "legacy_rows", - &fields(&["id", "created_at", "body"]), - "created_at,id", - ); - let read_row = |offset| { - connection - .query_row(&query, [1024_i64, offset], |row| { - Ok((row.get::<_, Option>(0)?, row.get::<_, bool>(3)?)) - }) - .unwrap() - }; - assert_eq!(read_row(0), (None, true)); - assert_eq!(read_row(1), (Some("later".into()), false)); -} - -#[test] -fn draft_read_consumes_at_most_the_limit_plus_one_bytes() { - let input = Cursor::new(vec![b'x'; 1024 * 1024]); - let mut input = input; - let bytes = read_bounded(&mut input, 17).unwrap(); - assert_eq!(bytes.len(), 18); - assert_eq!(input.position(), 18); -} - -#[test] -fn streaming_record_size_counter_stops_on_escape_expansion() { - let mut counter = SizeCounter { - limit: 32, - ..SizeCounter::default() - }; - let escaped = "\u{1}".repeat(1024); - assert!(serde_json::to_writer(&mut counter, &escaped).is_err()); - assert!(counter.exceeded); - assert!(counter.bytes <= counter.limit); -} - -#[tokio::test] -async fn reader_payload_reserves_envelope_space_for_the_actual_wire_request() { - let mut sender = BatchSender::default(); - let address = CallbackAddress { - destination: "d".repeat(1_800), - object_path: format!("/{}", "p".repeat(1_800)), - }; - let storage = CatalogStorage { callback: address }; - let envelope_size = serde_json::to_vec(&storage).unwrap().len(); - assert!(envelope_size < MAX_IMPORT_REQUEST_ENVELOPE_BYTES); - - let record = LegacyImportRecord::Flow(LegacyFlowRecord { - id: "near-limit".into(), - name: "legacy".into(), - graph_json: "g".repeat(MAX_IMPORT_RECORD_PAYLOAD_BYTES - 600), - enabled: true, - created_at: "2025-01-01T00:00:00Z".into(), - updated_at: "2025-01-01T00:00:00Z".into(), - last_run_at: None, - last_status: None, - require_approval: false, - }); - let captured = Arc::new(Mutex::new(Vec::new())); - let output = Arc::clone(&captured); - let mut send = move |batch| { - let output = Arc::clone(&output); - async move { - output.lock().unwrap().push(batch); - Ok(()) - } - }; - sender.push(record, &mut send).await.unwrap(); - sender.flush(&mut send).await.unwrap(); - let records = captured.lock().unwrap().remove(0); - assert!(matches!(&records[0], LegacyImportRecord::Flow(_))); - let request = ImportLegacyBatchRequest { storage, records }; - let encoded = serde_json::to_vec(&request).unwrap(); - assert!(encoded.len() <= MAX_IMPORT_BATCH_BYTES); - assert!(encoded.len() >= MAX_IMPORT_BATCH_BYTES - 1_000); -} diff --git a/crates/tinyflows-import/src/legacy_batch.rs b/crates/tinyflows-import/src/legacy_batch.rs deleted file mode 100644 index 5f462c14..00000000 --- a/crates/tinyflows-import/src/legacy_batch.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Shared semantic writer for legacy flow and checkpoint records. - -use std::collections::{BTreeMap, HashSet}; - -use anyhow::{Context, Result}; -use serde_json::Value; -use tinyflows_bus::catalog::{ - ImportLegacyBatchResponse, LegacyImportKind, LegacyImportProblem, LegacyImportRecord, - LegacyImportTally, -}; -use tinyflows_catalog::{ - Flow, FlowDraft, FlowRevision, FlowRun, FlowRunStep, FlowSuggestion, SuggestionStatus, -}; -use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; - -#[derive(Default)] -struct Counts { - written: u64, - existing: u64, - invalid: u64, -} - -/// Applies one already bounded batch through the caller's scoped catalog and checkpointer. -/// -/// Source parsing stays in the host-only reader; this shared writer owns legacy -/// semantic decoding, migration, idempotent catalog writes and checkpoint replay. -/// The host must run all batches during exclusive migration startup, before -/// run execution or other writers can mutate the target scope. Checkpoints use -/// the append-only sequence protocol and are not atomically fenced from a live -/// checkpointer writer. -pub async fn apply_legacy_batch( - records: &[LegacyImportRecord], - catalog: &FlowCatalogDocuments, - checkpointer: Option<&DriverCheckpointer>, -) -> Result { - let mut counts: BTreeMap = BTreeMap::new(); - let mut problems = Vec::new(); - let mut imported_flows = HashSet::new(); - let mut checkpoint_records = Vec::new(); - - for record in records { - match record { - LegacyImportRecord::Flow(row) => match decode_flow(row) { - Ok(flow) => match catalog.import_flow(&flow).await { - Ok(wrote) => { - tally(&mut counts, LegacyImportKind::Flow, wrote); - imported_flows.insert(flow.id); - } - Err(error) => return Err(error), - }, - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Flow, - Some(&row.id), - error.to_string(), - ), - }, - LegacyImportRecord::Revision(row) => { - let decoded = (|| -> Result { - let graph = - serde_json::from_str(&row.graph_json).context("graph_json is not JSON")?; - Ok(FlowRevision { - id: row.id.clone(), - flow_id: row.flow_id.clone(), - graph, - name: row.name.clone(), - require_approval: row.require_approval, - created_at: row.created_at.clone(), - }) - })(); - match decoded { - Ok(value) => { - if imported_flows.contains(&value.flow_id) - || catalog.get_flow(&value.flow_id).await?.is_some() - { - tally( - &mut counts, - LegacyImportKind::Revision, - catalog.import_revision(&value).await?, - ); - } else { - problem( - &mut counts, - &mut problems, - LegacyImportKind::Revision, - Some(&value.id), - "owner flow was not imported".into(), - ); - } - } - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Revision, - Some(&row.id), - error.to_string(), - ), - } - } - LegacyImportRecord::Run(row) => { - let decoded = (|| -> Result { - Ok(FlowRun { - id: row.id.clone(), - flow_id: row.flow_id.clone(), - thread_id: row.thread_id.clone(), - status: row.status.clone(), - started_at: row.started_at.clone(), - finished_at: row.finished_at.clone(), - steps: serde_json::from_str::>(&row.steps_json) - .context("run steps are corrupt")?, - pending_approvals: serde_json::from_str(&row.pending_approvals_json) - .context("pending approvals are corrupt")?, - error: row.error.clone(), - graph_hash: row.graph_hash.clone(), - }) - })(); - match decoded { - Ok(value) => { - if imported_flows.contains(&value.flow_id) - || catalog.get_flow(&value.flow_id).await?.is_some() - { - tally( - &mut counts, - LegacyImportKind::Run, - catalog.import_run(&value).await?, - ); - } else { - problem( - &mut counts, - &mut problems, - LegacyImportKind::Run, - Some(&value.id), - "owner flow was not imported".into(), - ); - } - } - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Run, - Some(&row.id), - error.to_string(), - ), - } - } - LegacyImportRecord::State(row) => { - match serde_json::from_str::(&row.value_json) { - Ok(value) => tally( - &mut counts, - LegacyImportKind::State, - catalog.import_kv(&row.namespace, &row.key, &value).await?, - ), - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::State, - Some(&row.key), - error.to_string(), - ), - } - } - LegacyImportRecord::Suggestion(row) => { - let decoded = (|| -> Result { - Ok(FlowSuggestion { - id: row.id.clone(), - title: row.title.clone(), - one_liner: row.one_liner.clone(), - rationale: row.rationale.clone(), - trigger_hint: row.trigger_hint.clone(), - steps_outline: serde_json::from_str(&row.steps_json) - .context("suggestion steps are corrupt")?, - suggested_connections: serde_json::from_str(&row.connections_json) - .context("suggestion connections are corrupt")?, - suggested_slugs: serde_json::from_str(&row.slugs_json) - .context("suggestion slugs are corrupt")?, - build_prompt: row.build_prompt.clone(), - confidence: row.confidence, - status: SuggestionStatus::from_str_lossy(&row.status), - created_at: row.created_at.clone(), - source_run_id: row.source_run_id.clone(), - }) - })(); - match decoded { - Ok(value) => tally( - &mut counts, - LegacyImportKind::Suggestion, - catalog.import_suggestion(&value).await?, - ), - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Suggestion, - Some(&row.id), - error.to_string(), - ), - } - } - LegacyImportRecord::Draft(row) => match serde_json::from_slice::(&row.bytes) - { - Ok(value) if safe_draft_id(&value.id) => tally( - &mut counts, - LegacyImportKind::Draft, - catalog.import_draft(&value).await?, - ), - Ok(value) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Draft, - Some(&row.file_name), - format!("'{}' is not a usable draft id", value.id), - ), - Err(error) => problem( - &mut counts, - &mut problems, - LegacyImportKind::Draft, - Some(&row.file_name), - error.to_string(), - ), - }, - LegacyImportRecord::Checkpoint { .. } - | LegacyImportRecord::SqliteCheckpointWrite(_) - | LegacyImportRecord::FileCheckpointWrite { .. } => { - checkpoint_records.push(record.clone()); - } - LegacyImportRecord::Invalid { - record_kind: LegacyImportKind::Checkpoint | LegacyImportKind::Write, - .. - } => { - checkpoint_records.push(record.clone()); - } - LegacyImportRecord::Invalid { - record_kind, - identity, - message, - } => problem( - &mut counts, - &mut problems, - *record_kind, - identity.as_deref(), - message.clone(), - ), - } - } - if !checkpoint_records.is_empty() { - let checkpointer = - checkpointer.context("checkpoint import requires a scoped checkpointer")?; - let response = crate::legacy_checkpoint::apply_legacy_checkpoint_batch( - &checkpoint_records, - checkpointer, - ) - .await?; - for tally in response.tallies { - let count = counts.entry(tally.kind).or_default(); - count.written += tally.written; - count.existing += tally.existing; - count.invalid += tally.invalid; - } - problems.extend(response.problems); - } - Ok(ImportLegacyBatchResponse { - tallies: counts - .into_iter() - .map(|(kind, count)| LegacyImportTally { - kind, - written: count.written, - existing: count.existing, - invalid: count.invalid, - }) - .collect(), - problems, - }) -} - -fn decode_flow(row: &tinyflows_bus::catalog::LegacyFlowRecord) -> Result { - let raw: Value = serde_json::from_str(&row.graph_json).context("graph_json is not JSON")?; - let migrated = tinyflows::migrate::migrate(raw).map_err(|error| anyhow::anyhow!("{error}"))?; - let graph = serde_json::from_value(migrated).context("graph_json does not decode")?; - Ok(Flow { - id: row.id.clone(), - name: row.name.clone(), - graph, - enabled: row.enabled, - created_at: row.created_at.clone(), - updated_at: row.updated_at.clone(), - last_run_at: row.last_run_at.clone(), - last_status: row.last_status.clone(), - require_approval: row.require_approval, - }) -} - -fn safe_draft_id(id: &str) -> bool { - !id.is_empty() - && id.len() <= 64 - && id.chars().all(|character| { - character.is_ascii_alphanumeric() || character == '-' || character == '_' - }) -} - -fn tally(counts: &mut BTreeMap, kind: LegacyImportKind, wrote: bool) { - let count = counts.entry(kind).or_default(); - if wrote { - count.written += 1 - } else { - count.existing += 1 - } -} - -fn problem( - counts: &mut BTreeMap, - problems: &mut Vec, - kind: LegacyImportKind, - identity: Option<&str>, - message: String, -) { - counts.entry(kind).or_default().invalid += 1; - problems.push(LegacyImportProblem { - kind, - identity: identity.map(|value| value.chars().take(128).collect()), - message: message.chars().take(240).collect(), - }); -} - -#[cfg(test)] -#[path = "legacy_batch_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/legacy_batch_tests.rs b/crates/tinyflows-import/src/legacy_batch_tests.rs deleted file mode 100644 index 28742c31..00000000 --- a/crates/tinyflows-import/src/legacy_batch_tests.rs +++ /dev/null @@ -1,125 +0,0 @@ -use super::*; -use crate::test_support::docs; -use serde_json::json; -use std::sync::Arc; -use tinyflows::graph::ids::NodeId; -use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer}; -use tinyflows_bus::catalog::{LegacyDraftRecord, LegacyImportRecord}; - -fn stores() -> (FlowCatalogDocuments, DriverCheckpointer) { - let docs = docs(); - ( - FlowCatalogDocuments::new(Arc::clone(&docs)), - DriverCheckpointer::new(docs), - ) -} - -fn checkpoint(thread_id: &str, checkpoint_id: &str) -> Checkpoint { - Checkpoint { - thread_id: thread_id.into(), - checkpoint_id: checkpoint_id.into(), - run_id: Some("run-owner".into()), - parent_checkpoint_id: None, - namespace: vec![], - state: json!({"resume":"legacy"}), - next_nodes: vec![NodeId::new("next")], - completed_tasks: vec![], - pending_writes: vec![], - interrupts: vec![], - pending_activations: None, - barrier_arrivals: vec![], - metadata: json!({"imported":true}), - } -} - -#[tokio::test] -async fn invalid_draft_is_reported_and_does_not_stop_later_records() { - let (catalog, checkpointer) = stores(); - let invalid = serde_json::to_vec(&FlowDraft { - id: "../unsafe".into(), - flow_id: None, - name: "unsafe".into(), - graph: json!({}), - origin: tinyflows_catalog::DraftOrigin::Canvas, - created_at: "2025-01-01T00:00:00Z".into(), - updated_at: "2025-01-01T00:00:00Z".into(), - }) - .unwrap(); - let good = FlowDraft { - id: "good_draft".into(), - flow_id: None, - name: "Good".into(), - graph: json!({}), - origin: tinyflows_catalog::DraftOrigin::Canvas, - created_at: "2025-01-01T00:00:00Z".into(), - updated_at: "2025-01-01T00:00:00Z".into(), - }; - let records = vec![ - LegacyImportRecord::Draft(LegacyDraftRecord { - file_name: "unsafe.json".into(), - bytes: invalid, - }), - LegacyImportRecord::Draft(LegacyDraftRecord { - file_name: "good_draft.json".into(), - bytes: serde_json::to_vec(&good).unwrap(), - }), - ]; - let response = apply_legacy_batch(&records, &catalog, Some(&checkpointer)) - .await - .unwrap(); - assert_eq!(response.tallies[0].invalid, 1); - assert_eq!(response.tallies[0].written, 1); - assert_eq!(response.problems.len(), 1); -} - -#[tokio::test] -async fn pending_writes_import_in_a_later_batch_and_never_replace_existing_values() { - let (catalog, checkpointer) = stores(); - let saved = checkpoint("different-thread-than-run-id", "checkpoint-1"); - let request = [LegacyImportRecord::Checkpoint { - raw_json: serde_json::to_string(&saved).unwrap(), - }]; - let checkpoint_result = apply_legacy_batch(&request, &catalog, Some(&checkpointer)) - .await - .unwrap(); - assert_eq!(checkpoint_result.tallies[0].written, 1); - - let write = |payload| LegacyImportRecord::FileCheckpointWrite { - thread_id: saved.thread_id.clone(), - raw_json: serde_json::to_string(&json!({ - "namespace": [], - "checkpoint_id": saved.checkpoint_id.clone(), - "write": { - "node": "next", - "task_id": "task-1", - "idx": 0, - "channel": "result", - "payload": payload - } - })) - .unwrap(), - }; - let first = apply_legacy_batch( - &[write(json!({"value":"original"}))], - &catalog, - Some(&checkpointer), - ) - .await - .unwrap(); - assert_eq!(first.tallies[0].written, 1); - let replay = apply_legacy_batch( - &[write(json!({"value":"newer-live-value"}))], - &catalog, - Some(&checkpointer), - ) - .await - .unwrap(); - assert_eq!(replay.tallies[0].existing, 1); - let config = CheckpointConfig { - thread_id: saved.thread_id, - checkpoint_id: Some(saved.checkpoint_id), - namespace: vec![], - }; - let writes = checkpointer.get_writes(&config).await.unwrap(); - assert_eq!(writes[0].payload, json!({"value":"original"})); -} diff --git a/crates/tinyflows-import/src/legacy_checkpoint.rs b/crates/tinyflows-import/src/legacy_checkpoint.rs deleted file mode 100644 index 63994711..00000000 --- a/crates/tinyflows-import/src/legacy_checkpoint.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! Shared semantic importer for checkpoint and pending-write records. - -use std::collections::BTreeMap; - -use anyhow::{Context, Result}; -use serde::Deserialize; -use serde_json::Value; -use tinyflows::graph::{Checkpoint, CheckpointConfig, Checkpointer, PendingWrite}; -use tinyflows_bus::catalog::{ - ImportLegacyBatchResponse, LegacyImportKind, LegacyImportProblem, LegacyImportRecord, - LegacyImportTally, -}; -use tinyflows_drivers::DriverCheckpointer; - -#[derive(Default)] -struct Counts { - written: u64, - existing: u64, - invalid: u64, -} - -/// Imports raw checkpoint/write DTOs through the shared checkpointer writer. -/// -/// Like the original one-time importer, this requires exclusive migration -/// startup access to the target scope before live run writers start. -pub async fn apply_legacy_checkpoint_batch( - records: &[LegacyImportRecord], - checkpointer: &DriverCheckpointer, -) -> Result { - let mut counts = BTreeMap::::new(); - let mut problems = Vec::new(); - let mut checkpoints = Vec::new(); - let mut writes = Vec::new(); - for record in records { - match record { - LegacyImportRecord::Checkpoint { raw_json } => { - match serde_json::from_str::>(raw_json) { - Ok(checkpoint) => checkpoints.push(checkpoint), - Err(error) => invalid( - &mut counts, - &mut problems, - LegacyImportKind::Checkpoint, - None, - error.to_string(), - ), - } - } - LegacyImportRecord::SqliteCheckpointWrite(row) => { - let decoded = (|| -> Result<_> { - Ok(( - CheckpointConfig { - thread_id: row.thread_id.clone(), - checkpoint_id: Some(row.checkpoint_id.clone()), - namespace: serde_json::from_str(&row.namespace_json) - .context("write namespace is corrupt")?, - }, - PendingWrite { - node: tinyflows::graph::ids::NodeId::new(row.node.clone()), - task_id: row.task_id.clone(), - idx: row.idx, - channel: row.channel.clone(), - payload: serde_json::from_str(&row.payload_json) - .context("write payload is corrupt")?, - }, - )) - })(); - match decoded { - Ok(value) => writes.push(value), - Err(error) => invalid( - &mut counts, - &mut problems, - LegacyImportKind::Write, - Some(&row.task_id), - error.to_string(), - ), - } - } - LegacyImportRecord::FileCheckpointWrite { - thread_id, - raw_json, - } => match serde_json::from_str::(raw_json) { - Ok(row) => writes.push(( - CheckpointConfig { - thread_id: thread_id.clone(), - checkpoint_id: Some(row.checkpoint_id), - namespace: row.namespace, - }, - row.write, - )), - Err(error) => invalid( - &mut counts, - &mut problems, - LegacyImportKind::Write, - Some(thread_id), - error.to_string(), - ), - }, - LegacyImportRecord::Invalid { - record_kind, - identity, - message, - } if matches!( - record_kind, - LegacyImportKind::Checkpoint | LegacyImportKind::Write - ) => - { - invalid( - &mut counts, - &mut problems, - *record_kind, - identity.as_deref(), - message.clone(), - ); - } - _ => { - return Err(anyhow::anyhow!( - "checkpoint batch contains a non-checkpoint record" - )); - } - } - } - replay(checkpointer, checkpoints, writes, &mut counts).await?; - Ok(ImportLegacyBatchResponse { - tallies: counts - .into_iter() - .map(|(kind, count)| LegacyImportTally { - kind, - written: count.written, - existing: count.existing, - invalid: count.invalid, - }) - .collect(), - problems, - }) -} - -#[derive(Deserialize)] -struct FileWrite { - #[serde(default)] - namespace: Vec, - checkpoint_id: String, - write: PendingWrite, -} - -fn invalid( - counts: &mut BTreeMap, - problems: &mut Vec, - kind: LegacyImportKind, - identity: Option<&str>, - message: String, -) { - counts.entry(kind).or_default().invalid += 1; - problems.push(LegacyImportProblem { - kind, - identity: identity.map(|value| value.chars().take(128).collect()), - message: message.chars().take(240).collect(), - }); -} - -async fn replay( - checkpointer: &DriverCheckpointer, - checkpoints: Vec>, - writes: Vec<(CheckpointConfig, PendingWrite)>, - counts: &mut BTreeMap, -) -> Result<()> { - for checkpoint in checkpoints { - let held = checkpointer - .get_scoped( - &checkpoint.thread_id, - Some(&checkpoint.checkpoint_id), - &checkpoint.namespace, - ) - .await - .map_err(|error| anyhow::anyhow!("{error}"))? - .is_some(); - if held { - counts - .entry(LegacyImportKind::Checkpoint) - .or_default() - .existing += 1; - continue; - } - checkpointer - .put(checkpoint) - .await - .map_err(|error| anyhow::anyhow!("{error}"))?; - counts - .entry(LegacyImportKind::Checkpoint) - .or_default() - .written += 1; - } - let mut groups: Vec<(CheckpointConfig, Vec)> = Vec::new(); - for (config, write) in writes { - match groups.iter_mut().find(|(known, _)| *known == config) { - Some((_, group)) => group.push(write), - None => groups.push((config, vec![write])), - } - } - for (config, group) in groups { - let (written, existing) = checkpointer - .import_writes_if_absent(&config, &group) - .await - .map_err(|error| anyhow::anyhow!("{error}"))?; - let count = counts.entry(LegacyImportKind::Write).or_default(); - count.written += written as u64; - count.existing += existing as u64; - } - Ok(()) -} diff --git a/crates/tinyflows-import/src/legacy_cron.rs b/crates/tinyflows-import/src/legacy_cron.rs deleted file mode 100644 index 40458eaa..00000000 --- a/crates/tinyflows-import/src/legacy_cron.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! Reads the legacy `jobs.db` (the cron store before it moved to documents) into -//! plain bus records. It names only the transport vocabulary -//! (`tinyflows_bus::schedule::types`), so a host that reaches the scheduler -//! through the bus can replay the file through the module's -//! `import_job` / `import_run` store operations without linking the scheduler -//! implementation. -//! -//! Two tables, `cron_jobs` and `cron_runs`. `cron_jobs` gained most of its -//! columns over time, so a database from before a column existed reads as if -//! the column held its default. An older `cron_job_policies` side table, which -//! the old store folded into `cron_jobs.retries` / `single_flight` on its next -//! open, is read the same way here. - -use std::collections::HashSet; -use std::path::Path; - -use anyhow::{Context, Result, anyhow}; -use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use tinyflows_bus::schedule::types::{ - CronJob, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobPolicy, JobType, Schedule, - SessionTarget, -}; - -use crate::legacy_db::{LegacyDb, column_or}; -use crate::report::ImportReport; - -/// What `jobs.db` held, decoded. -#[derive(Debug, Default)] -pub struct LegacyCron { - /// Every decodable job, oldest first. - pub jobs: Vec, - /// Every decodable run, in row order. Its `job_id` is the job's id in the - /// legacy file; a flow-schedule job is filed under another id by the - /// target store, so a caller remaps it from the id the import returns. - pub runs: Vec, - /// Rows that could not be decoded; also counted in the report as invalid. - pub report: ImportReport, -} - -/// Reads `jobs_db` (the file the old store was opened with). A missing file or -/// one without a `cron_jobs` table reads as empty. -/// -/// # Errors -/// -/// When the file exists but cannot be read. A row that cannot be decoded is -/// counted in the report as invalid. -pub async fn read_legacy_cron(jobs_db: &Path) -> Result { - let mut legacy = LegacyCron::default(); - let Some(db) = LegacyDb::open(jobs_db)? else { - return Ok(legacy); - }; - let tables = db.tables().await?; - if !tables.contains("cron_jobs") { - return Ok(legacy); - } - legacy.jobs = read_jobs(&db, &tables, &mut legacy.report).await?; - if tables.contains("cron_runs") { - legacy.runs = read_runs(&db, &mut legacy.report).await?; - } - Ok(legacy) -} - -/// One `cron_jobs` row with the raw text columns still raw. -struct JobRow { - id: String, - expression: String, - command: String, - schedule: Option, - job_type: String, - prompt: Option, - name: Option, - session_target: String, - model: Option, - enabled: i64, - delivery: Option, - delete_after_run: i64, - created_at: String, - next_run: String, - last_run: Option, - last_status: Option, - last_output: Option, - agent_id: Option, - origin: Option, - retries: Option, - single_flight: i64, - claim_generation: i64, -} - -async fn read_jobs( - db: &LegacyDb, - tables: &HashSet, - report: &mut ImportReport, -) -> Result> { - let have = db.columns("cron_jobs").await?; - let select = format!( - "SELECT id, expression, command, {}, {}, {}, {}, {}, {}, {}, {}, {}, created_at, next_run, \ - last_run, last_status, last_output, {}, {}, {}, {}, {} FROM cron_jobs ORDER BY created_at, id", - column_or(&have, "schedule", "NULL"), - column_or(&have, "job_type", "'shell'"), - column_or(&have, "prompt", "NULL"), - column_or(&have, "name", "NULL"), - column_or(&have, "session_target", "'isolated'"), - column_or(&have, "model", "NULL"), - column_or(&have, "enabled", "1"), - column_or(&have, "delivery", "NULL"), - column_or(&have, "delete_after_run", "0"), - column_or(&have, "agent_id", "NULL"), - column_or(&have, "origin", "NULL"), - column_or(&have, "retries", "NULL"), - column_or(&have, "single_flight", "0"), - column_or(&have, "claim_generation", "0"), - ); - let mut rows = db - .read("cron jobs", move |conn| { - let mut statement = conn.prepare(&select)?; - statement - .query_map([], |row| { - Ok(JobRow { - id: row.get(0)?, - expression: row.get(1)?, - command: row.get(2)?, - schedule: row.get(3)?, - job_type: row.get(4)?, - prompt: row.get(5)?, - name: row.get(6)?, - session_target: row.get(7)?, - model: row.get(8)?, - enabled: row.get(9)?, - delivery: row.get(10)?, - delete_after_run: row.get(11)?, - created_at: row.get(12)?, - next_run: row.get(13)?, - last_run: row.get(14)?, - last_status: row.get(15)?, - last_output: row.get(16)?, - agent_id: row.get(17)?, - origin: row.get(18)?, - retries: row.get(19)?, - single_flight: row.get(20)?, - claim_generation: row.get(21)?, - }) - })? - .collect::, _>>() - }) - .await?; - if tables.contains("cron_job_policies") { - let policies = read_policies(db).await?; - for row in &mut rows { - if let Some((retries, single_flight)) = policies.get(&row.id) { - row.retries = *retries; - row.single_flight = *single_flight; - } - } - } - let mut jobs = Vec::new(); - for row in rows { - let id = row.id.clone(); - match decode_job(row) { - Ok(job) => jobs.push(job), - Err(error) => report.invalid("jobs", format!("job '{id}': {error:#}")), - } - } - Ok(jobs) -} - -/// `cron_job_policies`: `job_id -> (retries, single_flight)`. -async fn read_policies(db: &LegacyDb) -> Result, i64)>> { - db.read("cron job policies", |conn| { - let mut statement = - conn.prepare("SELECT job_id, retries, single_flight FROM cron_job_policies")?; - statement - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, (row.get(1)?, row.get(2)?))) - })? - .collect::, _>>() - }) - .await -} - -fn decode_job(row: JobRow) -> Result { - let schedule = match row.schedule.as_deref().map(str::trim) { - Some(raw) if !raw.is_empty() => serde_json::from_str(raw) - .with_context(|| format!("cron schedule JSON does not parse: {raw}"))?, - _ if row.expression.trim().is_empty() => { - return Err(anyhow!("missing schedule and legacy expression")); - } - // A job written before `schedule` existed carried only its expression. - _ => Schedule::Cron { - expr: row.expression.clone(), - tz: None, - active_hours: None, - }, - }; - let delivery = match row.delivery.as_deref().map(str::trim) { - Some(raw) if !raw.is_empty() => serde_json::from_str(raw) - .with_context(|| format!("cron delivery JSON does not parse: {raw}"))?, - _ => DeliveryConfig::default(), - }; - let origin = match row.origin.as_deref().map(str::trim) { - Some(raw) if !raw.is_empty() => Some( - serde_json::from_str::(raw) - .with_context(|| format!("cron origin JSON does not parse: {raw}"))?, - ), - _ => None, - }; - Ok(CronJob { - claim_generation: u64::try_from(row.claim_generation) - .context("claim generation is negative")?, - policy: JobPolicy { - retries: row - .retries - .map(|n| u32::try_from(n.max(0)).unwrap_or(u32::MAX)), - single_flight: row.single_flight != 0, - }, - id: row.id, - expression: row.expression, - schedule, - command: row.command, - job_type: JobType::parse(&row.job_type), - prompt: row.prompt, - name: row.name, - session_target: SessionTarget::parse(&row.session_target), - model: row.model, - agent_id: row.agent_id, - enabled: row.enabled != 0, - delivery, - delete_after_run: row.delete_after_run != 0, - created_at: instant(&row.created_at)?, - next_run: instant(&row.next_run)?, - last_run: row.last_run.as_deref().map(instant).transpose()?, - last_status: row.last_status, - last_output: row.last_output, - origin, - }) -} - -fn instant(raw: &str) -> Result> { - Ok(DateTime::parse_from_rfc3339(raw) - .with_context(|| format!("invalid RFC3339 timestamp: {raw}"))? - .with_timezone(&Utc)) -} - -async fn read_runs(db: &LegacyDb, report: &mut ImportReport) -> Result> { - let have = db.columns("cron_runs").await?; - let select = format!( - "SELECT id, job_id, started_at, finished_at, status, output, duration_ms, {} \ - FROM cron_runs ORDER BY id", - column_or(&have, "delivery_status", "NULL"), - ); - let rows = db - .read("cron runs", move |conn| { - let mut statement = conn.prepare(&select)?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - row.get::<_, Option>(7)?, - )) - })? - .collect::, _>>() - }) - .await?; - let mut runs = Vec::new(); - for (id, job_id, started_at, finished_at, status, output, duration_ms, delivery_status) in rows - { - match (instant(&started_at), instant(&finished_at)) { - (Ok(started_at), Ok(finished_at)) => runs.push(CronRun { - id, - job_id, - run_id: None, - started_at, - finished_at, - status, - output, - duration_ms, - delivery_status: delivery_status.as_deref().and_then(DeliveryStatus::parse), - }), - (Err(error), _) | (_, Err(error)) => { - report.invalid("runs", format!("run {id}: {error:#}")); - } - } - } - Ok(runs) -} diff --git a/crates/tinyflows-import/src/legacy_db.rs b/crates/tinyflows-import/src/legacy_db.rs deleted file mode 100644 index 9cc9f616..00000000 --- a/crates/tinyflows-import/src/legacy_db.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Reading a legacy SQLite file without touching it. -//! -//! The file is copied, with its `-wal` and `-shm` if it has them, to a scratch -//! directory, and the copy is opened through the SQLite driver's native handle. -//! The driver creates what it needs and sets WAL mode on open, so opening the -//! original would modify it; the copy takes that instead and is deleted when -//! the [`LegacyDb`] drops. - -use std::collections::HashSet; -use std::path::Path; - -use anyhow::{Context, Result}; -use tinystoragedrivers_sqlite::SqliteNative; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -/// A scratch copy of a legacy SQLite database, open for reading. -pub(crate) struct LegacyDb { - // Held for its drop: the directory goes when the handle does. - _scratch: tempfile::TempDir, - native: SqliteNative, -} - -impl LegacyDb { - /// Opens a copy of the database at `path`; `None` when there is no such - /// file, which is the normal state of a store that was never used. - pub(crate) fn open(path: &Path) -> Result> { - if !path.is_file() { - return Ok(None); - } - let scratch = tempfile::tempdir().context("creating a scratch directory")?; - let name = path - .file_name() - .context("a database path has a file name")? - .to_os_string(); - for suffix in ["", "-wal", "-shm"] { - let mut from = path.as_os_str().to_os_string(); - from.push(suffix); - let from = std::path::PathBuf::from(from); - if from.is_file() { - let mut to = name.clone(); - to.push(suffix); - std::fs::copy(&from, scratch.path().join(to)) - .with_context(|| format!("copying {}", from.display()))?; - } - } - let native = SqliteNative::open(scratch.path().join(&name)) - .with_context(|| format!("opening a copy of {}", path.display()))?; - Ok(Some(Self { - _scratch: scratch, - native, - })) - } - - /// Runs `read` on the copy's connection. - pub(crate) async fn read(&self, what: &'static str, read: F) -> Result - where - T: Send + 'static, - F: FnOnce(&Connection) -> tinystoragedrivers_sqlite::rusqlite::Result + Send + 'static, - { - self.native - .with_connection(read) - .await - .with_context(|| format!("reading {what} from the legacy database")) - } - - /// The tables the database has. - pub(crate) async fn tables(&self) -> Result> { - self.read("the table list", |conn| { - let mut statement = - conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")?; - let names = statement - .query_map([], |row| row.get::<_, String>(0))? - .collect::, _>>()?; - Ok(names) - }) - .await - } - - /// The columns `table` has; empty when it has none or does not exist. - pub(crate) async fn columns(&self, table: &'static str) -> Result> { - self.read("a column list", move |conn| { - let mut statement = conn.prepare(&format!("PRAGMA table_info({table})"))?; - let names = statement - .query_map([], |row| row.get::<_, String>(1))? - .collect::, _>>()?; - Ok(names) - }) - .await - } -} - -/// `column` when `have` lists it, else the literal `default`, as a select-list -/// entry: how a database written before a column existed is read as if the -/// column held its default. -pub(crate) fn column_or(have: &HashSet, column: &str, default: &str) -> String { - if have.contains(column) { - column.to_string() - } else { - format!("{default} AS {column}") - } -} - -#[cfg(test)] -#[path = "legacy_db_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/legacy_db_tests.rs b/crates/tinyflows-import/src/legacy_db_tests.rs deleted file mode 100644 index 0ee5e5c4..00000000 --- a/crates/tinyflows-import/src/legacy_db_tests.rs +++ /dev/null @@ -1,116 +0,0 @@ -use super::*; -use tinystoragedrivers_sqlite::rusqlite::Connection; - -fn build(path: &Path) { - let conn = Connection::open(path).unwrap(); - conn.execute_batch( - "CREATE TABLE things (id TEXT PRIMARY KEY, label TEXT); - INSERT INTO things VALUES ('a', 'first');", - ) - .unwrap(); -} - -#[tokio::test] -async fn a_missing_file_is_no_database_rather_than_an_error() { - let dir = tempfile::tempdir().unwrap(); - assert!( - LegacyDb::open(&dir.path().join("nope.db")) - .unwrap() - .is_none() - ); -} - -#[tokio::test] -async fn tables_and_columns_are_listed_from_a_copy() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("legacy.db"); - build(&path); - let db = LegacyDb::open(&path).unwrap().unwrap(); - assert!(db.tables().await.unwrap().contains("things")); - let columns = db.columns("things").await.unwrap(); - assert!(columns.contains("id") && columns.contains("label")); - assert!(db.columns("absent").await.unwrap().is_empty()); -} - -#[tokio::test] -async fn reading_leaves_the_original_byte_for_byte_alone() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("legacy.db"); - build(&path); - let before = std::fs::read(&path).unwrap(); - let listing_before: Vec<_> = std::fs::read_dir(dir.path()) - .unwrap() - .map(|e| e.unwrap().file_name()) - .collect(); - - let db = LegacyDb::open(&path).unwrap().unwrap(); - let labels: Vec = db - .read("labels", |conn| { - let mut statement = conn.prepare("SELECT label FROM things")?; - statement - .query_map([], |row| row.get(0))? - .collect::>() - }) - .await - .unwrap(); - drop(db); - - assert_eq!(labels, ["first"]); - assert_eq!(std::fs::read(&path).unwrap(), before); - let listing_after: Vec<_> = std::fs::read_dir(dir.path()) - .unwrap() - .map(|e| e.unwrap().file_name()) - .collect(); - assert_eq!(listing_after, listing_before, "no -wal or -shm appeared"); -} - -#[tokio::test] -async fn rows_that_only_live_in_the_wal_are_read() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("legacy.db"); - build(&path); - // Keep a connection open so the WAL is not folded back into the main file. - let writer = Connection::open(&path).unwrap(); - writer.pragma_update(None, "journal_mode", "WAL").unwrap(); - writer.pragma_update(None, "wal_autocheckpoint", 0).unwrap(); - writer - .execute("INSERT INTO things VALUES ('b', 'in the wal')", []) - .unwrap(); - assert!(dir.path().join("legacy.db-wal").is_file()); - - let db = LegacyDb::open(&path).unwrap().unwrap(); - let count: i64 = db - .read("count", |conn| { - conn.query_row("SELECT COUNT(*) FROM things", [], |row| row.get(0)) - }) - .await - .unwrap(); - assert_eq!(count, 2); -} - -#[tokio::test] -async fn a_file_that_is_not_a_database_is_an_error_naming_the_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("junk.db"); - std::fs::write( - &path, - b"this is not sqlite at all, just text, long enough to be a header.....", - ) - .unwrap(); - let outcome = match LegacyDb::open(&path) { - Err(error) => error, - Ok(Some(db)) => db.tables().await.unwrap_err(), - Ok(None) => panic!("the file exists"), - }; - assert!( - format!("{outcome:#}").contains("junk.db") - || format!("{outcome:#}").contains("legacy database") - ); -} - -#[test] -fn a_missing_column_reads_as_its_default() { - let have: HashSet = ["id".to_string()].into(); - assert_eq!(column_or(&have, "id", "0"), "id"); - assert_eq!(column_or(&have, "extra", "NULL"), "NULL AS extra"); -} diff --git a/crates/tinyflows-import/src/lib.rs b/crates/tinyflows-import/src/lib.rs deleted file mode 100644 index 70be4cc5..00000000 --- a/crates/tinyflows-import/src/lib.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! One-time, read-only import of the stores tinyflows hosts kept before storage -//! moved onto the `tinystoragedrivers` ports. -//! -//! Each `import_*` function reads one kind of legacy store and writes what it -//! held into the matching port-backed store from `tinyflows-drivers`: -//! -//! | Legacy | Function | Target | -//! | --- | --- | --- | -//! | `flows.db` (definitions, revisions, runs, state, suggestions) and `drafts/*.json` | [`import_flow_catalog`] | [`FlowCatalogDocuments`] | -//! | `jobs.db` (cron jobs and runs) | [`import_cron`] | [`CronDocuments`] | -//! | `checkpoints.db` | [`import_sqlite_checkpoints`] | [`DriverCheckpointer`] | -//! | `FileCheckpointer` directory (`*.jsonl`) | [`import_file_checkpoints`] | [`DriverCheckpointer`] | -//! | `FileStateStore` directory | [`import_file_state`] | [`FlowCatalogDocuments`] | -//! | `FileWorkflowStore` (definitions, runs, journal, proposals, revisions) | [`import_workflow_store`] | [`DriverWorkflowStore`] | -//! -//! # Guarantees -//! -//! - **Read-only.** Legacy files are never modified. A SQLite file is copied -//! (with its `-wal` and `-shm`) to a scratch directory and read there, so the -//! driver's own opening of it (WAL mode, a busy timeout) never touches the -//! original. JSON and JSONL files are only read. -//! - **Id-preserving.** Every record lands under the id it had. The exception -//! is a flow-schedule cron job, which the document store files under the -//! deterministic id it derives from the flow (see -//! [`CronDocuments::import_job`]); its runs follow it. -//! - **Only if absent.** Nothing already in the target is replaced, so the -//! import can be repeated after a failure, or run against a scope that has -//! been written to since, without clobbering anything. The report counts what -//! was written and what was already there. -//! - **Forgiving about records, strict about files.** A record that cannot be -//! decoded (a graph from a newer build, a corrupt row) is counted as invalid, -//! logged and skipped, so one bad row costs only itself. A file that cannot -//! be opened at all is an error, and the host decides whether to retry. A -//! store that is not there is simply an empty report. -//! -//! # Features -//! -//! `schedule`, `catalog`, `engine`, `store`, and `checkpoint-read` plus -//! `checkpoint-write` are enabled by default. Hosts that only need to read -//! legacy records for a compiled module use `flow-read`, `checkpoint-read`, or -//! `schedule-read` without linking TinyFlows implementation crates. The -//! `flow-write` and `checkpoint-write` features provide shared semantic -//! writers for the owning module and established importer APIs. -//! -//! The host decides when to run this and records that it has (a marker in its -//! own storage); this crate keeps no state of its own. -//! -//! [`FlowCatalogDocuments`]: tinyflows_drivers::catalog::FlowCatalogDocuments -//! [`CronDocuments`]: tinyflows_drivers::schedule::CronDocuments -//! [`DriverCheckpointer`]: tinyflows_drivers::DriverCheckpointer -//! [`DriverWorkflowStore`]: tinyflows_drivers::store::DriverWorkflowStore - -#[cfg(feature = "catalog")] -mod catalog; -#[cfg(feature = "checkpoint-read")] -mod checkpoint_reader; -#[cfg(all( - feature = "engine", - feature = "checkpoint-read", - feature = "checkpoint-write" -))] -mod checkpoints; -#[cfg(feature = "schedule")] -mod cron; -#[cfg(feature = "flow-read")] -mod flow_reader; -#[cfg(feature = "flow-write")] -mod legacy_batch; -#[cfg(feature = "checkpoint-write")] -mod legacy_checkpoint; -#[cfg(feature = "schedule-read")] -mod legacy_cron; -#[cfg(any( - feature = "flow-read", - feature = "schedule-read", - feature = "checkpoint-read" -))] -mod legacy_db; -mod report; -#[cfg(feature = "catalog")] -mod state; -#[cfg(feature = "store")] -mod workflow_store; - -#[cfg(all( - test, - any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-read", - feature = "checkpoint-read", - feature = "schedule-read", - feature = "flow-write", - feature = "checkpoint-write" - ) -))] -mod test_support; - -#[cfg(feature = "catalog")] -pub use catalog::import_flow_catalog; -#[cfg(feature = "checkpoint-read")] -pub use checkpoint_reader::{ - read_legacy_checkpoint_batches, read_legacy_file_checkpoint_batches, - read_legacy_sqlite_checkpoint_batches, -}; -#[cfg(all( - feature = "engine", - feature = "checkpoint-read", - feature = "checkpoint-write" -))] -pub use checkpoints::{import_file_checkpoints, import_sqlite_checkpoints}; -#[cfg(feature = "schedule")] -pub use cron::import_cron; -#[cfg(feature = "flow-read")] -pub use flow_reader::read_legacy_flow_catalog; -#[cfg(feature = "flow-write")] -pub use legacy_batch::apply_legacy_batch; -#[cfg(feature = "checkpoint-write")] -pub use legacy_checkpoint::apply_legacy_checkpoint_batch; -#[cfg(feature = "schedule-read")] -pub use legacy_cron::{LegacyCron, read_legacy_cron}; -pub use report::{ImportReport, Tally}; -#[cfg(feature = "catalog")] -pub use state::import_file_state; -#[cfg(feature = "store")] -pub use workflow_store::{FileStoreLayout, import_workflow_store}; diff --git a/crates/tinyflows-import/src/report.rs b/crates/tinyflows-import/src/report.rs deleted file mode 100644 index 46ed8fbc..00000000 --- a/crates/tinyflows-import/src/report.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! What an import did, record kind by record kind. - -use std::collections::BTreeMap; - -#[cfg(all( - feature = "flow-read", - any(feature = "flow-write", feature = "checkpoint-write") -))] -fn legacy_kind_name(kind: tinyflows_bus::catalog::LegacyImportKind) -> &'static str { - use tinyflows_bus::catalog::LegacyImportKind as Kind; - match kind { - Kind::Flow => "flows", - Kind::Revision => "revisions", - Kind::Run => "runs", - Kind::State => "state", - Kind::Suggestion => "suggestions", - Kind::Draft => "drafts", - Kind::Checkpoint => "checkpoints", - Kind::Write => "writes", - } -} - -/// How many records of one kind an import wrote, found already present, or -/// could not decode. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct Tally { - /// Records this import wrote. - pub written: usize, - /// Records the target already held, left exactly as they were. - pub existing: usize, - /// Records in the legacy store that could not be decoded or were refused. - pub invalid: usize, -} - -/// The outcome of one `import_*` call. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct ImportReport { - kinds: BTreeMap<&'static str, Tally>, - problems: Vec, -} - -impl ImportReport { - #[cfg(all( - feature = "flow-read", - any(feature = "flow-write", feature = "checkpoint-write") - ))] - pub(crate) fn merge_legacy_batch( - &mut self, - response: tinyflows_bus::catalog::ImportLegacyBatchResponse, - ) { - for tally in response.tallies { - let kind = legacy_kind_name(tally.kind); - let total = self.kinds.entry(kind).or_default(); - total.written += tally.written as usize; - total.existing += tally.existing as usize; - total.invalid += tally.invalid as usize; - } - for problem in response.problems { - let identity = problem.identity.map_or_else(String::new, |id| { - let label = match problem.kind { - tinyflows_bus::catalog::LegacyImportKind::Flow => "flow", - tinyflows_bus::catalog::LegacyImportKind::Revision => "revision", - tinyflows_bus::catalog::LegacyImportKind::Run => "run", - tinyflows_bus::catalog::LegacyImportKind::State => "state", - tinyflows_bus::catalog::LegacyImportKind::Suggestion => "suggestion", - tinyflows_bus::catalog::LegacyImportKind::Draft => "draft", - tinyflows_bus::catalog::LegacyImportKind::Checkpoint => "checkpoint", - tinyflows_bus::catalog::LegacyImportKind::Write => "write", - }; - format!(" {label} '{id}'") - }); - let message = format!( - "{}{}: {}", - legacy_kind_name(problem.kind), - identity, - problem.message - ); - tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}"); - self.problems.push(message); - } - } - - pub fn wrote(&mut self, kind: &'static str, written: bool) { - let tally = self.kinds.entry(kind).or_default(); - if written { - tally.written += 1; - } else { - tally.existing += 1; - } - } - - /// Records one record that could not be imported, with why. - pub fn invalid(&mut self, kind: &'static str, what: impl std::fmt::Display) { - self.kinds.entry(kind).or_default().invalid += 1; - let message = format!("{kind}: {what}"); - tracing::warn!(target: "tinyflows_import", "[import] skipped an unreadable record: {message}"); - self.problems.push(message); - } - - /// The tally of `kind` (`"flows"`, `"runs"`, `"jobs"`, ...); all zero when - /// the store held none. - pub fn tally(&self, kind: &str) -> Tally { - self.kinds.get(kind).copied().unwrap_or_default() - } - - /// How many records of `kind` this import wrote. - pub fn written(&self, kind: &str) -> usize { - self.tally(kind).written - } - - /// Every kind seen, with its tally, in name order. - pub fn kinds(&self) -> impl Iterator + '_ { - self.kinds.iter().map(|(kind, tally)| (*kind, *tally)) - } - - /// One line per record that could not be imported. - pub fn problems(&self) -> &[String] { - &self.problems - } - - /// Records written across every kind. - pub fn total_written(&self) -> usize { - self.kinds.values().map(|tally| tally.written).sum() - } - - /// Whether every legacy record was either written or already present. - pub fn is_clean(&self) -> bool { - self.problems.is_empty() - } -} - -#[cfg(test)] -#[path = "report_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/report_tests.rs b/crates/tinyflows-import/src/report_tests.rs deleted file mode 100644 index 6c3647dd..00000000 --- a/crates/tinyflows-import/src/report_tests.rs +++ /dev/null @@ -1,40 +0,0 @@ -use super::*; - -#[test] -fn an_empty_report_is_clean_and_all_zero() { - let report = ImportReport::default(); - assert!(report.is_clean()); - assert_eq!(report.tally("flows"), Tally::default()); - assert_eq!(report.total_written(), 0); - assert_eq!(report.kinds().count(), 0); -} - -#[test] -fn written_and_existing_are_counted_per_kind() { - let mut report = ImportReport::default(); - report.wrote("flows", true); - report.wrote("flows", true); - report.wrote("flows", false); - report.wrote("runs", true); - assert_eq!( - report.tally("flows"), - Tally { - written: 2, - existing: 1, - invalid: 0 - } - ); - assert_eq!(report.written("runs"), 1); - assert_eq!(report.total_written(), 3); - let kinds: Vec<_> = report.kinds().map(|(kind, _)| kind).collect(); - assert_eq!(kinds, ["flows", "runs"]); -} - -#[test] -fn an_invalid_record_is_counted_and_named() { - let mut report = ImportReport::default(); - report.invalid("flows", "flow 'f' graph does not decode"); - assert!(!report.is_clean()); - assert_eq!(report.tally("flows").invalid, 1); - assert_eq!(report.problems(), ["flows: flow 'f' graph does not decode"]); -} diff --git a/crates/tinyflows-import/src/state.rs b/crates/tinyflows-import/src/state.rs deleted file mode 100644 index da69b56c..00000000 --- a/crates/tinyflows-import/src/state.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! The retired `FileStateStore`: one JSON file per key. -//! -//! A key was hashed into its file name, so the files cannot be listed back into -//! keys: the old store keeps no index. The caller names the keys it wants (a -//! host knows which state keys its flows use), and each is looked up by the -//! same hash the old store used. - -use std::path::Path; - -use anyhow::{Context, Result}; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use tinyflows_drivers::catalog::FlowCatalogDocuments; - -use crate::report::ImportReport; - -/// The values stored under `keys` in `namespace` beneath `root`, the directory -/// a `FileStateStore` was created with, copied into `catalog`'s state under the -/// same namespace and key (where `FlowStateDocuments` reads them). A key that -/// has no file is not an error and is not counted. -/// -/// # Errors -/// -/// When a state file exists but cannot be read, or the target store fails. A -/// file that is not JSON is counted in the report as invalid. -pub async fn import_file_state( - root: &Path, - namespace: &str, - keys: &[String], - catalog: &FlowCatalogDocuments, -) -> Result { - let mut report = ImportReport::default(); - let dir = root.join(digest(namespace)); - for key in keys { - let path = dir.join(format!("{}.json", digest(key))); - let body = match std::fs::read(&path) { - Ok(body) => body, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(error).with_context(|| format!("reading {}", path.display())), - }; - match serde_json::from_slice::(&body) { - Ok(value) => report.wrote("state", catalog.import_kv(namespace, key, &value).await?), - Err(error) => report.invalid("state", format!("{namespace}/{key}: {error}")), - } - } - Ok(report) -} - -/// The hex SHA-256 the old store turned a namespace or key into a path -/// component with. -fn digest(value: &str) -> String { - hex::encode(Sha256::digest(value.as_bytes())) -} - -#[cfg(test)] -#[path = "state_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/state_tests.rs b/crates/tinyflows-import/src/state_tests.rs deleted file mode 100644 index 5cd2978d..00000000 --- a/crates/tinyflows-import/src/state_tests.rs +++ /dev/null @@ -1,140 +0,0 @@ -use super::*; -use crate::test_support::{docs, expected, fixtures, tree_digest}; -use serde_json::json; - -fn target() -> FlowCatalogDocuments { - FlowCatalogDocuments::new(docs()) -} - -fn keys() -> (String, Vec) { - let expected = expected(); - let state = &expected["state"]; - ( - state["namespace"].as_str().unwrap().to_string(), - state["keys"] - .as_array() - .unwrap() - .iter() - .map(|key| key.as_str().unwrap().to_string()) - .collect(), - ) -} - -#[tokio::test] -async fn the_named_keys_come_across_under_their_namespace() { - let (namespace, keys) = keys(); - let catalog = target(); - let report = import_file_state(&fixtures().join("file-state"), &namespace, &keys, &catalog) - .await - .unwrap(); - assert!(report.is_clean()); - assert_eq!( - report.tally("state").written, - 2, - "`never-written` has no file" - ); - assert_eq!( - catalog.kv_get(&namespace, "cursor").await.unwrap(), - Some(json!({ "page": 9 })) - ); - assert_eq!( - catalog - .kv_get(&namespace, "weird/key with spaces") - .await - .unwrap(), - Some(json!([1, 2, 3])) - ); - assert_eq!( - catalog.kv_get(&namespace, "never-written").await.unwrap(), - None - ); -} - -#[tokio::test] -async fn a_key_in_the_wrong_namespace_finds_nothing() { - let (_, keys) = keys(); - let report = import_file_state( - &fixtures().join("file-state"), - "workflow:other", - &keys, - &target(), - ) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); -} - -#[tokio::test] -async fn importing_again_keeps_what_the_state_store_holds() { - let (namespace, keys) = keys(); - let catalog = target(); - catalog - .kv_set(&namespace, "cursor", &json!("newer")) - .await - .unwrap(); - let report = import_file_state(&fixtures().join("file-state"), &namespace, &keys, &catalog) - .await - .unwrap(); - assert_eq!(report.tally("state").existing, 1); - assert_eq!(report.tally("state").written, 1); - assert_eq!( - catalog.kv_get(&namespace, "cursor").await.unwrap(), - Some(json!("newer")) - ); -} - -#[tokio::test] -async fn the_legacy_files_are_left_exactly_as_they_were() { - let (namespace, keys) = keys(); - let before = tree_digest(&fixtures()); - import_file_state(&fixtures().join("file-state"), &namespace, &keys, &target()) - .await - .unwrap(); - assert_eq!(tree_digest(&fixtures()), before); -} - -#[tokio::test] -async fn a_state_file_that_is_not_json_is_counted_not_fatal() { - let root = tempfile::tempdir().unwrap(); - let dir = root.path().join(digest("ns")); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join(format!("{}.json", digest("bad"))), "{oops").unwrap(); - std::fs::write(dir.join(format!("{}.json", digest("good"))), "42").unwrap(); - let catalog = target(); - let report = import_file_state(root.path(), "ns", &["bad".into(), "good".into()], &catalog) - .await - .unwrap(); - assert_eq!(report.tally("state").invalid, 1); - assert_eq!(report.tally("state").written, 1); - assert_eq!(catalog.kv_get("ns", "good").await.unwrap(), Some(json!(42))); -} - -#[tokio::test] -async fn a_key_whose_file_cannot_be_read_is_an_error() { - let root = tempfile::tempdir().unwrap(); - let dir = root.path().join(digest("ns")); - std::fs::create_dir_all(dir.join(format!("{}.json", digest("k")))).unwrap(); - let outcome = import_file_state(root.path(), "ns", &["k".into()], &target()).await; - assert!(outcome.is_err()); -} - -#[tokio::test] -async fn no_keys_and_no_directory_are_empty_reports() { - let root = tempfile::tempdir().unwrap(); - let report = import_file_state(root.path(), "ns", &[], &target()) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); - let report = import_file_state(&root.path().join("none"), "ns", &["k".into()], &target()) - .await - .unwrap(); - assert_eq!(report.total_written(), 0); -} - -#[test] -fn the_digest_is_the_hex_sha256_the_old_store_used() { - assert_eq!( - digest("abc"), - "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" - ); -} diff --git a/crates/tinyflows-import/src/test_support.rs b/crates/tinyflows-import/src/test_support.rs deleted file mode 100644 index d0e6992c..00000000 --- a/crates/tinyflows-import/src/test_support.rs +++ /dev/null @@ -1,185 +0,0 @@ -//! Fixtures the import tests share. -//! -//! `tests/fixtures` holds one store of every legacy kind, written by the -//! retired writers (`tinyflows-sqlite`, `FileCheckpointer`, `FileStateStore`, -//! `FileWorkflowStore`), and `expected.json`, the same data read back through -//! those stores' own readers. The committed files are never written to: tests -//! that import from them also check that no file under the tree changed. - -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -use std::collections::BTreeMap; -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-read", - feature = "checkpoint-read" -))] -use std::path::{Path, PathBuf}; -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-write" -))] -use std::sync::Arc; - -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -use serde_json::Value; -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -use sha2::{Digest, Sha256}; -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-write" -))] -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; - -/// The committed legacy stores. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-read", - feature = "checkpoint-read" -))] -pub(crate) fn fixtures() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures") -} - -/// What the legacy readers saw in [`fixtures`]. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -pub(crate) fn expected() -> Value { - let body = std::fs::read(fixtures().join("expected.json")).expect("expected.json"); - canonical(serde_json::from_slice(&body).expect("expected.json parses")) -} - -/// `value` with every RFC 3339 string rewritten to millisecond precision in -/// UTC: storage keeps one epoch-millisecond field per instant, so a -/// comparison with the legacy fixtures is made at that precision. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -pub(crate) fn canonical(value: Value) -> Value { - match value { - Value::String(text) => match chrono::DateTime::parse_from_rfc3339(&text) { - Ok(at) => Value::String( - at.with_timezone(&chrono::Utc) - .to_rfc3339_opts(chrono::SecondsFormat::Millis, true), - ), - Err(_) => Value::String(text), - }, - Value::Array(items) => Value::Array(items.into_iter().map(canonical).collect()), - Value::Object(map) => { - Value::Object(map.into_iter().map(|(k, v)| (k, canonical(v))).collect()) - } - other => other, - } -} - -/// A document handle under `scope` of a fresh in-memory backend. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-write" -))] -pub(crate) fn docs() -> Arc { - docs_in(&MemoryStorage::new(), "local") -} - -/// `storage`'s documents under `scope`. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store", - feature = "flow-write" -))] -pub(crate) fn docs_in(storage: &MemoryStorage, scope: &str) -> Arc { - Arc::clone( - storage - .for_scope(&Scope::new(scope).unwrap()) - .unwrap() - .documents(), - ) -} - -/// Every file under `dir` and the digest of its bytes, by relative path. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -pub(crate) fn tree_digest(dir: &Path) -> BTreeMap { - let mut digests = BTreeMap::new(); - let mut pending = vec![dir.to_path_buf()]; - while let Some(next) = pending.pop() { - for entry in std::fs::read_dir(&next).unwrap() { - let path = entry.unwrap().path(); - if path.is_dir() { - pending.push(path); - } else { - let relative = path - .strip_prefix(dir) - .unwrap() - .to_string_lossy() - .into_owned(); - digests.insert( - relative, - hex::encode(Sha256::digest(std::fs::read(&path).unwrap())), - ); - } - } - } - digests -} - -/// The JSON form, for the model types that are not `PartialEq`. -#[cfg(any( - feature = "engine", - feature = "catalog", - feature = "schedule", - feature = "store" -))] -pub(crate) fn json(value: &T) -> Value { - canonical(serde_json::to_value(value).unwrap()) -} - -/// `items` sorted by their `key` string, for comparing lists whose order the -/// two stores define differently. -#[cfg(feature = "catalog")] -pub(crate) fn sorted_by(mut items: Vec, key: &str) -> Vec { - items.sort_by(|a, b| a[key].as_str().cmp(&b[key].as_str())); - items -} diff --git a/crates/tinyflows-import/src/workflow_store.rs b/crates/tinyflows-import/src/workflow_store.rs deleted file mode 100644 index 798a6626..00000000 --- a/crates/tinyflows-import/src/workflow_store.rs +++ /dev/null @@ -1,283 +0,0 @@ -//! The retired `FileWorkflowStore`: workflows as JSON documents, with their -//! runs, journal, proposals and revisions kept beside them. -//! -//! Where each lives is the old store's configuration, not something the files -//! say, so [`FileStoreLayout`] names the directories; [`FileStoreLayout::with_state`] -//! and [`FileStoreLayout::discover`] derive them the way the old constructors -//! did. Only the write layer's definitions are imported: the lower layers of a -//! layered store were repository-provided defaults, authored files a host -//! loads from the repository itself, not state the store owned. - -use std::path::{Component, Path, PathBuf}; - -use anyhow::{Context, Result}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use tinyflows::store::parse_workflow; -use tinyflows::store::types::{ - RunRecord, WorkflowNote, WorkflowProposal, WorkflowRecord, WorkflowRevision, -}; -use tinyflows_drivers::store::DriverWorkflowStore; - -use crate::report::ImportReport; - -/// Where an old `FileWorkflowStore` kept each kind of record. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileStoreLayout { - /// The write directory: one `.json` document per workflow. - pub definitions: PathBuf, - /// One `.json` per run record. - pub runs: PathBuf, - /// One `.json` per workflow: its notes, as a JSON array. - pub journal: PathBuf, - /// One `.json` per proposal. - pub proposals: PathBuf, - /// Directories of superseded copies, each holding - /// `/.json`. More than one when a store was - /// upgraded from one that kept them beside the definitions. - pub revisions: Vec, -} - -impl FileStoreLayout { - /// The layout of `FileWorkflowStore::with_state(vec![definitions], state_dir)`: - /// runs, journal and proposals under `state_dir`, revisions under the - /// definition state the write directory hashes to, and any older - /// `.revisions` kept beside the definitions. - pub fn with_state(definitions: impl Into, state_dir: &Path) -> Self { - let definitions = definitions.into(); - Self { - runs: state_dir.join("runs"), - journal: state_dir.join("journal"), - proposals: state_dir.join("proposals"), - revisions: revisions_dirs(state_dir, &definitions), - definitions, - } - } - - /// The layout of `FileWorkflowStore::discover(home, cwd, _)`: definitions in - /// `/workflows`, definition state under `/state/workflows`, and - /// the runs, journal and proposals of the workspace `cwd` under that - /// directory's `scopes/`. - pub fn discover(home: &Path, cwd: &Path) -> Self { - let state_root = home.join("state").join("workflows"); - let definitions = home.join("workflows"); - let run_state = state_root.join("scopes").join(workspace_scope(cwd)); - Self { - runs: run_state.join("runs"), - journal: run_state.join("journal"), - proposals: run_state.join("proposals"), - revisions: revisions_dirs(&state_root, &definitions), - definitions, - } - } -} - -/// The current and legacy places a write directory's revisions can be. -fn revisions_dirs(state_root: &Path, definitions: &Path) -> Vec { - vec![ - definition_state_dir(state_root, definitions).join("revisions"), - definitions.join(".revisions"), - ] -} - -/// The state the old store shared between every store writing one catalog: -/// `/definitions/`. -fn definition_state_dir(state_root: &Path, write_dir: &Path) -> PathBuf { - let identity = canonical_path_identity(&absolute(write_dir)); - let scope = hex::encode(Sha256::digest(identity.as_os_str().as_encoded_bytes())); - state_root.join("definitions").join(scope) -} - -/// The directory name that identified one workspace: the first sixteen hex -/// characters of the digest of its canonical path. -fn workspace_scope(workspace: &Path) -> String { - let identity = std::fs::canonicalize(workspace).unwrap_or_else(|_| absolute(workspace)); - let digest = hex::encode(Sha256::digest(identity.to_string_lossy().as_bytes())); - digest[..16].to_string() -} - -fn absolute(path: &Path) -> PathBuf { - if path.is_absolute() { - return path.to_path_buf(); - } - std::env::current_dir() - .unwrap_or_else(|_| PathBuf::from(".")) - .join(path) -} - -/// Resolves existing symlinks while keeping lexical meaning for parts that do -/// not exist yet, as the old store did to name a catalog. -fn canonical_path_identity(path: &Path) -> PathBuf { - let mut resolved = PathBuf::new(); - for component in path.components() { - match component { - Component::CurDir => {} - Component::ParentDir => { - if resolved.exists() { - resolved = std::fs::canonicalize(&resolved).unwrap_or(resolved); - } - resolved.pop(); - } - other => { - resolved.push(other.as_os_str()); - if resolved.exists() { - resolved = std::fs::canonicalize(&resolved).unwrap_or(resolved); - } - } - } - } - resolved -} - -/// A revision file: when the copy stopped being current, and the copy. -#[derive(Deserialize)] -struct StoredRevision { - superseded_at: u64, - record: WorkflowRecord, -} - -/// The workflows, revisions, runs, journals and proposals under `layout`, -/// copied into `store`. -/// -/// # Errors -/// -/// When a directory exists but cannot be listed, a file cannot be read, or the -/// target store fails. A file that cannot be parsed, or a workflow the engine -/// would not compile, is counted in the report as invalid. -pub async fn import_workflow_store( - layout: &FileStoreLayout, - store: &DriverWorkflowStore, -) -> Result { - let mut report = ImportReport::default(); - - for (path, id) in json_files(&layout.definitions)? { - let text = std::fs::read_to_string(&path) - .with_context(|| format!("reading {}", path.display()))?; - match parse_workflow(&text, &id) { - Ok(record) => match store.import_workflow(&record).await { - Ok(written) => report.wrote("workflows", written), - Err(tinyflows::store::types::WorkflowError::Storage(message)) => { - anyhow::bail!("storage: {message}") - } - Err(error) => report.invalid("workflows", format!("{}: {error}", path.display())), - }, - Err(error) => report.invalid("workflows", format!("{}: {error}", path.display())), - } - } - - for revisions in &layout.revisions { - let Some(workflows) = list_dir(revisions)? else { - continue; - }; - for workflow_dir in workflows.into_iter().filter(|path| path.is_dir()) { - for (path, id) in json_files(&workflow_dir)? { - let Some(stored) = read_json::(&path, "revisions", &mut report)? - else { - continue; - }; - let revision = WorkflowRevision { - id, - superseded_at: stored.superseded_at, - record: stored.record, - }; - report.wrote("revisions", store.import_revision(&revision).await?); - } - } - } - - for (path, _) in json_files(&layout.runs)? { - if let Some(run) = read_json::(&path, "runs", &mut report)? { - report.wrote("runs", store.import_run(&run).await?); - } - } - - for (path, workflow_id) in json_files(&layout.journal)? { - let Some(notes) = read_json::>(&path, "notes", &mut report)? else { - continue; - }; - let offered = notes - .iter() - .filter(|n| n.workflow_id == workflow_id) - .count(); - let added = store.import_notes(&workflow_id, ¬es).await?; - for _ in 0..added { - report.wrote("notes", true); - } - for _ in added..offered { - report.wrote("notes", false); - } - for _ in offered..notes.len() { - report.invalid( - "notes", - format!("{}: a note belongs to another workflow", path.display()), - ); - } - } - - for (path, _) in json_files(&layout.proposals)? { - if let Some(proposal) = read_json::(&path, "proposals", &mut report)? { - report.wrote("proposals", store.import_proposal(&proposal).await?); - } - } - - tracing::info!( - target: "tinyflows_import", - written = report.total_written(), - problems = report.problems().len(), - "[import] workflow store" - ); - Ok(report) -} - -/// The entries of `dir`, sorted; `None` when it does not exist. -fn list_dir(dir: &Path) -> Result>> { - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error).with_context(|| format!("listing {}", dir.display())), - }; - let mut paths: Vec = entries - .filter_map(|entry| entry.ok().map(|entry| entry.path())) - .collect(); - paths.sort(); - Ok(Some(paths)) -} - -/// Every `.json` file directly in `dir`, with its stem, sorted. A missing -/// directory is the normal state of a store that never held that record kind. -fn json_files(dir: &Path) -> Result> { - Ok(list_dir(dir)? - .unwrap_or_default() - .into_iter() - .filter(|path| { - path.is_file() - && path - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) - }) - .filter_map(|path| { - let stem = path.file_stem()?.to_str()?.to_string(); - Some((path, stem)) - }) - .collect()) -} - -/// Reads and parses one file; an unparseable one is counted, not an error. -fn read_json Deserialize<'de>>( - path: &Path, - kind: &'static str, - report: &mut ImportReport, -) -> Result> { - let body = std::fs::read(path).with_context(|| format!("reading {}", path.display()))?; - match serde_json::from_slice(&body) { - Ok(value) => Ok(Some(value)), - Err(error) => { - report.invalid(kind, format!("{}: {error}", path.display())); - Ok(None) - } - } -} - -#[cfg(test)] -#[path = "workflow_store_tests.rs"] -mod tests; diff --git a/crates/tinyflows-import/src/workflow_store_tests.rs b/crates/tinyflows-import/src/workflow_store_tests.rs deleted file mode 100644 index ec29a60a..00000000 --- a/crates/tinyflows-import/src/workflow_store_tests.rs +++ /dev/null @@ -1,337 +0,0 @@ -use super::*; -use crate::test_support::{docs, docs_in, expected, fixtures, json, tree_digest}; -use serde_json::{Value, json}; -use tinyflows::store::WorkflowStore; -use tinystoragedrivers_core::MemoryStorage; - -fn base() -> PathBuf { - fixtures().join("workflow-store") -} - -/// The layout the committed store was written with. The revisions directory is -/// named by a hash of the path it was generated at, so it is found, not derived. -fn layout() -> FileStoreLayout { - let hashed = std::fs::read_dir(base().join("state/definitions")) - .unwrap() - .next() - .expect("one definition state directory") - .unwrap() - .path(); - FileStoreLayout { - definitions: base().join("workflows"), - runs: base().join("state/runs"), - journal: base().join("state/journal"), - proposals: base().join("state/proposals"), - revisions: vec![ - hashed.join("revisions"), - base().join("workflows/.revisions"), - ], - } -} - -fn target() -> DriverWorkflowStore { - DriverWorkflowStore::new(docs()) -} - -/// `value` without the path a record was read from, which no store keeps. -fn portable(mut value: Value) -> Value { - fn strip(value: &mut Value) { - match value { - Value::Object(map) => { - map.remove("source_path"); - map.values_mut().for_each(strip); - } - Value::Array(items) => items.iter_mut().for_each(strip), - _ => {} - } - } - strip(&mut value); - value -} - -#[tokio::test] -async fn the_whole_store_comes_across_with_its_ids_and_every_field() { - let store = target(); - let report = import_workflow_store(&layout(), &store).await.unwrap(); - assert!(report.is_clean(), "{:?}", report.problems()); - let expected = expected(); - let expected = &expected["workflow_store"]; - - // Definitions: the legacy store's `load()`, id for id. - for legacy in expected["workflows"].as_array().unwrap() { - let id = legacy["id"].as_str().unwrap(); - let imported = store - .get(id) - .unwrap() - .unwrap_or_else(|| panic!("{id} imported")); - assert_eq!( - portable(json(&imported)), - portable(legacy.clone()), - "workflow {id}" - ); - } - assert_eq!(store.list().unwrap().len(), 2); - // Revisions, including the one a delete left behind. - for (id, legacy) in expected["revisions"].as_object().unwrap() { - let imported = store.list_revisions(id).unwrap(); - assert_eq!( - portable(Value::Array(imported.iter().map(json).collect())), - portable(legacy.clone()), - "revisions of {id}" - ); - } - assert!(store.get("retired").unwrap().is_none()); - assert_eq!(store.list_revisions("retired").unwrap().len(), 1); - // Runs. - for (id, legacy) in expected["runs"].as_object().unwrap() { - let imported = store.list_runs(id).unwrap(); - assert_eq!( - Value::Array(imported.iter().map(json).collect()), - *legacy, - "runs of {id}" - ); - } - assert_eq!(store.unsettled_runs().unwrap().len(), 1); - // Notes and proposals. - let notes = store.list_notes("sweep").unwrap(); - assert_eq!( - Value::Array(notes.iter().map(json).collect()), - expected["notes"] - ); - let proposals = store.list_proposals("sweep").unwrap(); - assert_eq!( - Value::Array(proposals.iter().map(json).collect()), - expected["proposals"] - ); -} - -#[tokio::test] -async fn the_report_counts_every_kind() { - let report = import_workflow_store(&layout(), &target()).await.unwrap(); - for (kind, written) in [ - ("workflows", 2), - ("revisions", 3), - ("runs", 3), - ("notes", 2), - ("proposals", 1), - ] { - assert_eq!(report.tally(kind).written, written, "{kind}"); - } -} - -#[tokio::test] -async fn importing_again_writes_nothing() { - let store = target(); - let first = import_workflow_store(&layout(), &store).await.unwrap(); - let second = import_workflow_store(&layout(), &store).await.unwrap(); - assert_eq!(second.total_written(), 0); - for (kind, tally) in first.kinds() { - assert_eq!(second.tally(kind).existing, tally.written, "{kind}"); - } - assert_eq!(store.list_revisions("sweep").unwrap().len(), 2); -} - -#[tokio::test] -async fn the_legacy_files_are_left_exactly_as_they_were() { - let before = tree_digest(&fixtures()); - import_workflow_store(&layout(), &target()).await.unwrap(); - assert_eq!(tree_digest(&fixtures()), before); -} - -#[tokio::test] -async fn nothing_the_target_already_holds_is_replaced() { - let storage = MemoryStorage::new(); - let store = DriverWorkflowStore::new(docs_in(&storage, "local")); - let mut live = tinyflows::store::parse_workflow( - &json!({ - "id": "sweep", "name": "Live", - "nodes": [{ "id": "t", "kind": "trigger", "name": "start", "config": { "trigger_kind": "manual" } }], - "edges": [] - }) - .to_string(), - "sweep", - ) - .unwrap(); - live.description = "live description".into(); - store.save(&live).unwrap(); - - let report = import_workflow_store(&layout(), &store).await.unwrap(); - - assert_eq!(report.tally("workflows").existing, 1); - assert_eq!( - store.get("sweep").unwrap().unwrap().description, - "live description" - ); -} - -#[tokio::test] -async fn files_that_do_not_parse_are_counted_and_the_rest_imported() { - let root = tempfile::tempdir().unwrap(); - let layout = - FileStoreLayout::with_state(root.path().join("workflows"), &root.path().join("state")); - let write = |path: PathBuf, body: &str| { - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - std::fs::write(path, body).unwrap(); - }; - let good = json!({ - "name": "Good", - "nodes": [{ "id": "t", "kind": "trigger", "name": "start", "config": { "trigger_kind": "manual" } }], - "edges": [] - }) - .to_string(); - // An id from the file name, since the document has none. - write(layout.definitions.join("good.json"), &good); - write(layout.definitions.join("junk.json"), "{nope"); - write( - layout.definitions.join("hollow.json"), - r#"{"nodes":[],"edges":[]}"#, - ); - write(layout.definitions.join("notes.txt"), "ignored"); - write(layout.runs.join("bad.json"), "{nope"); - write(layout.journal.join("w.json"), "not an array"); - write(layout.proposals.join("bad.json"), "[]"); - write(layout.revisions[0].join("good").join("bad.json"), "{nope"); - - let store = target(); - let report = import_workflow_store(&layout, &store).await.unwrap(); - - assert_eq!(report.tally("workflows").written, 1); - assert_eq!(report.tally("workflows").invalid, 2); - for kind in ["runs", "notes", "proposals", "revisions"] { - assert_eq!(report.tally(kind).invalid, 1, "{kind}"); - } - assert_eq!(store.get("good").unwrap().unwrap().name, "Good"); -} - -#[tokio::test] -async fn notes_filed_under_another_workflow_are_counted_invalid() { - let root = tempfile::tempdir().unwrap(); - let layout = FileStoreLayout::with_state(root.path().join("w"), &root.path().join("s")); - std::fs::create_dir_all(&layout.journal).unwrap(); - use tinyflows::store::types::{NoteKind, NoteSource, WorkflowNote}; - let note = |workflow: &str, id: &str| WorkflowNote { - id: id.to_string(), - workflow_id: workflow.to_string(), - kind: NoteKind::Observation, - text: "t".to_string(), - recorded_at: 1, - source: NoteSource::System, - run_ids: Vec::new(), - superseded_by: None, - pinned: false, - }; - let body = serde_json::to_vec(&[note("mine", "a"), note("other", "b")]).unwrap(); - std::fs::write(layout.journal.join("mine.json"), body).unwrap(); - let store = target(); - let report = import_workflow_store(&layout, &store).await.unwrap(); - assert_eq!(report.tally("notes").written, 1); - assert_eq!(report.tally("notes").invalid, 1); - assert_eq!(store.list_notes("mine").unwrap().len(), 1); - assert!(store.list_notes("other").unwrap().is_empty()); -} - -#[tokio::test] -async fn an_empty_or_absent_store_is_an_empty_report() { - let root = tempfile::tempdir().unwrap(); - let layout = FileStoreLayout::with_state(root.path().join("w"), &root.path().join("s")); - let report = import_workflow_store(&layout, &target()).await.unwrap(); - assert_eq!(report.total_written(), 0); - assert!(report.is_clean()); -} - -#[tokio::test] -async fn a_definitions_path_that_is_a_file_is_an_error() { - let root = tempfile::tempdir().unwrap(); - let file = root.path().join("workflows"); - std::fs::write(&file, "x").unwrap(); - let layout = FileStoreLayout::with_state(file, &root.path().join("s")); - assert!(import_workflow_store(&layout, &target()).await.is_err()); -} - -#[test] -fn with_state_puts_each_kind_where_the_old_constructor_did() { - let root = tempfile::tempdir().unwrap(); - let state = root.path().join("state"); - let definitions = root.path().join("workflows"); - let layout = FileStoreLayout::with_state(&definitions, &state); - assert_eq!(layout.definitions, definitions); - assert_eq!(layout.runs, state.join("runs")); - assert_eq!(layout.journal, state.join("journal")); - assert_eq!(layout.proposals, state.join("proposals")); - assert_eq!(layout.revisions.len(), 2); - let hashed = &layout.revisions[0]; - assert_eq!(hashed.file_name().unwrap(), "revisions"); - let scope = hashed.parent().unwrap(); - assert_eq!(scope.parent().unwrap(), state.join("definitions")); - assert_eq!(scope.file_name().unwrap().len(), 64, "a hex SHA-256"); - assert_eq!(layout.revisions[1], definitions.join(".revisions")); -} - -#[test] -fn the_revisions_directory_does_not_depend_on_how_the_write_directory_is_spelled() { - let root = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(root.path().join("workflows")).unwrap(); - std::fs::create_dir_all(root.path().join("elsewhere")).unwrap(); - let state = root.path().join("state"); - let plain = FileStoreLayout::with_state(root.path().join("workflows"), &state); - let roundabout = FileStoreLayout::with_state( - root.path().join("elsewhere").join("..").join("workflows"), - &state, - ); - assert_eq!(plain.revisions[0], roundabout.revisions[0]); - let other = FileStoreLayout::with_state(root.path().join("elsewhere"), &state); - assert_ne!(plain.revisions[0], other.revisions[0]); -} - -#[cfg(unix)] -#[test] -fn a_symlinked_write_directory_names_the_same_state_as_its_target() { - let root = tempfile::tempdir().unwrap(); - let real = root.path().join("real"); - std::fs::create_dir_all(&real).unwrap(); - let link = root.path().join("link"); - std::os::unix::fs::symlink(&real, &link).unwrap(); - let state = root.path().join("state"); - assert_eq!( - FileStoreLayout::with_state(&real, &state).revisions[0], - FileStoreLayout::with_state(&link, &state).revisions[0] - ); -} - -#[test] -fn a_relative_write_directory_resolves_against_the_working_directory() { - let state = tempfile::tempdir().unwrap(); - let relative = FileStoreLayout::with_state("relative-workflows", state.path()); - let absolute = FileStoreLayout::with_state( - std::env::current_dir().unwrap().join("relative-workflows"), - state.path(), - ); - assert_eq!(relative.revisions[0], absolute.revisions[0]); -} - -#[test] -fn discover_scopes_runs_to_the_workspace_and_definitions_to_home() { - let home = tempfile::tempdir().unwrap(); - let workspace = tempfile::tempdir().unwrap(); - let layout = FileStoreLayout::discover(home.path(), workspace.path()); - let state = home.path().join("state/workflows"); - assert_eq!(layout.definitions, home.path().join("workflows")); - let scope = layout.runs.parent().unwrap(); - assert_eq!(scope.parent().unwrap(), state.join("scopes")); - assert_eq!(scope.file_name().unwrap().len(), 16); - assert_eq!(layout.journal, scope.join("journal")); - assert_eq!(layout.proposals, scope.join("proposals")); - assert!(layout.revisions[0].starts_with(state.join("definitions"))); - // A different workspace gets its own run state but the same definition state. - let other = tempfile::tempdir().unwrap(); - let elsewhere = FileStoreLayout::discover(home.path(), other.path()); - assert_ne!(layout.runs, elsewhere.runs); - assert_eq!(layout.revisions, elsewhere.revisions); -} - -#[test] -fn a_workspace_that_does_not_exist_yet_still_has_a_scope() { - let home = tempfile::tempdir().unwrap(); - let layout = FileStoreLayout::discover(home.path(), Path::new("/no/such/workspace/yet")); - assert_eq!(layout.runs.parent().unwrap().file_name().unwrap().len(), 16); -} diff --git a/crates/tinyflows-import/tests/catalog_feature_contract_tests.rs b/crates/tinyflows-import/tests/catalog_feature_contract_tests.rs deleted file mode 100644 index 854b2761..00000000 --- a/crates/tinyflows-import/tests/catalog_feature_contract_tests.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! Compile and exercise the checkpoint importer APIs provided by `catalog`. -#![cfg(feature = "catalog")] - -use std::sync::Arc; - -use serde_json::Value; -use tinyflows_drivers::DriverCheckpointer; -use tinyflows_import::{import_file_checkpoints, import_sqlite_checkpoints}; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; - -#[tokio::test] -async fn catalog_feature_retains_checkpoint_import_entry_points() { - let directory = tempfile::tempdir().unwrap(); - let docs = Arc::clone( - MemoryStorage::new() - .for_scope(&Scope::new("local").unwrap()) - .unwrap() - .documents(), - ); - let checkpointer = DriverCheckpointer::::new(docs); - - let sqlite = import_sqlite_checkpoints( - &directory.path().join("missing-checkpoints.db"), - &checkpointer, - ) - .await - .unwrap(); - let files = import_file_checkpoints( - &directory.path().join("missing-file-checkpoints"), - &checkpointer, - ) - .await - .unwrap(); - - assert!(sqlite.is_clean()); - assert!(files.is_clean()); -} diff --git a/crates/tinyflows-import/tests/fixtures/README.md b/crates/tinyflows-import/tests/fixtures/README.md deleted file mode 100644 index 02c97ea6..00000000 --- a/crates/tinyflows-import/tests/fixtures/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Legacy store fixtures - -One store of every retired kind, written by the retired writers themselves -(tinyflows at `953d541`, v0.8.8: `tinyflows-sqlite`, `FileCheckpointer`, -`FileStateStore`, `FileWorkflowStore`), so the importer is tested against bytes -the old code produced and not against a format reconstructed from reading it. - -| Path | Written by | Holds | -| --- | --- | --- | -| `catalog/flows.db`, `catalog/drafts/*.json` | `tinyflows_sqlite::flows`, the draft store | two flows (one disabled and approval-gated), two revisions of `flow-alpha`, four runs (completed with a step, parked with an approval and a graph hash, live, failed), three state entries, two suggestions (one dismissed), two drafts | -| `cron/jobs.db` | `tinyflows_sqlite::schedule` | a named shell job, an agent job with a channel origin, a flow-schedule job, four runs with delivery statuses | -| `checkpoints/checkpoints.db`, `checkpoints/file/` | `SqliteCheckpointer`, `FileCheckpointer` | the same data in both: a thread with a subgraph namespace, a thread whose id needs escaping, three pending writes; the `file/` copy ends in a torn line | -| `file-state/` | `FileStateStore` | two keys under `workflow:flow-alpha` | -| `workflow-store/` | `FileWorkflowStore::with_state` | two workflows, three revisions (one left by a delete), three runs, a two-note journal with a supersession, a proposal | -| `expected.json` | the same stores' own readers | what each store returned when read back, which the tests compare the imported data to | - -Nothing here is ever written to: the tests that import from this tree hash every -file under it before and after. The `workflow-store/state/definitions/` -directory is named by a hash of the absolute path the fixture was generated at, -so the tests find it rather than derive it; the derivation itself is covered by -tests over temporary directories. diff --git a/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-1.json b/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-1.json deleted file mode 100644 index 8fcd20e9..00000000 --- a/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "id": "draft-1", - "name": "Scratch", - "graph": { - "nodes": [] - }, - "origin": "chat", - "created_at": "2025-03-01T12:00:00+00:00", - "updated_at": "2025-03-01T12:05:00+00:00" -} \ No newline at end of file diff --git a/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-2.json b/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-2.json deleted file mode 100644 index 9df8f40e..00000000 --- a/crates/tinyflows-import/tests/fixtures/catalog/drafts/draft-2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "id": "draft-2", - "flow_id": "flow-alpha", - "name": "Edit alpha", - "graph": { - "nodes": [ - { - "id": "x" - } - ] - }, - "origin": "canvas", - "created_at": "2025-03-01T13:00:00+00:00", - "updated_at": "2025-03-01T13:05:00+00:00" -} \ No newline at end of file diff --git a/crates/tinyflows-import/tests/fixtures/catalog/flows.db b/crates/tinyflows-import/tests/fixtures/catalog/flows.db deleted file mode 100644 index e6b8a0df058af8f7213207efb94d3f3ede20d513..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI5&u`<#700O`YgzuqHG-{RG=?a)!CI|kh?4EKUAU^Vt1Py0yshm85*UU+imm-N&J|8)I$(DDUYQ1SZtSV|<(=}T&bgTb5dV8a?x>XUjR$pGL2>rgo z%!m*;G^Ka3Re60&_}<37*H$;)5MHgkkqd>?<(AqF>x5jtza^|cSX&DRYnXC-ulfVa z&;?;YFjZ$oaWlWSzE!zf*$}!33U?~6tUg%V5~Og_HB*&sRjHEB4@$bzR-PnSQ?~4? z+0j{YOwe#%j<03QcE@sjdnj^(slM6KOtmVv+oo|KH-*O*ppZR@T~DM|vyqdmrYq`O zn(|iFsS#5>&@5Ja?xm)LT_?RdytjU*^1AT&Si=2vw}$=nW}JjHB`5T7WlvWo5~*x9 zayH-7AU{SpXa2nQ_#p@Aj-wfa7045;t3w0pFsMbnq`L|duBxD7UKaMhsJd__J(!;S z19bb9`x}*ech^}H61vrq6*elbR5mK>w=0|H+T@I;WLZt#sjO9~;oe@|yuEs-(o?zY zxA>vd{cMDDbjW(MNxgqn?%0NN-LFtmxI(X=438&LujV5sFNIam>NFav<#Zv{Uh$mB z0k4URH8Bt(5ZPD9fcH!sOUI(4-E!D58lgk`I3%j@d>V12^)efiDpotxq4HAC0cnyY{A{lZBk zB-w!IR?8g8dO$Y zeb~<`aO(>s5@OYtqtP*Is&j+uzsCnUmh$LOYBN8uV+rVd)$LjWE@RDPBDIo^oFv07 zn(nJsl?rXT_iMLDBDax?+i_oTzZ+rhGreT8yTe1NTj|L01(`WnbZ?^+Ejb`3FKjp) z&oTScuM8zp>2&1nqh2q;_eeLCYOs_J+Q4h?Z~_TlS&Ss?8FnPOeS)At*6VZ z^4&3nb9U{#HaqTp>dYRpthP*jUkGZL&c1r+VHpsJ9+*3B4>}-Fpn(W8e%?8Oa>xd{ zuhO-l)ZNTLBu-(v9TzM5(2rt?RD~Wwroz2QU_@2*Mkh{d47kTH?o$FGx-tE(CEz#X zrJ5~AFYR;#(LC7$F0t|Pe{<2~BQE)9?AjMS*hG3D00JNY0w4eaAOHeiI)R_O5E;!( zpRFXzo8@$pJ(T9tyZN>a%Xx+2N_w30@_%TU`#}+88hq2Mt^q=L zF?upss?9`^sR%2xG+Wx5mBr(MnG+IKq4`h;L!G9Ds*DOlK>0e?OmMnx$(qhYF}eY~ z%*i1b;&|t{SSZft3#EKP+L8*(1@`}&&Nq21Km)!{su^xmvNS+Vacvxnc^zv{UIc8O(yPBz%MY zV+6w*;)fboP0zB9NWppzDo@CR-topVsWT~AE|=58eZ9dItNFevMkf&N1RmQvT+;MM z=#~`a|HASApf4OC00JNY0w4eaAOHd&00JNY0w4eaPc4D*SUTcN?yyO{=y)s>nD}+a z|DSREXZ*jO+5$pw5C8!X009sH0T2KI5C8!X009sHfr|*lW5bbXES5~NwfQmrXB__r z{#Wz|2MB-w2!H?xfB*=900@8p2!H?xfWZHsfDnsIQ#aT$Aojw3*?((a>zI~dvPD>5 zkH$+k(%}%!5-v+s^=(?FloNNwotu$(X)1F;C^w-au}zD%XmQmsd;cHff5Gv8;y<82 zI6wddKmY_l00ck)1V8`;KmY_l00h1~0@E}t5D5?G*^l|%(R}PhniA;q^```)(=-#1 z3`cc5-MH-ie|-Em93P!HotUQ+93TJ!AOHd&00JNY0w4eaAOHeanLsl>n#tUpDwiiG zZ*XL~KVO$M+7qj1($C%6q3i5;`ep&iF682c_5yv6qH9TVduMxR*DzXZvC-+w*S=cj zCTV{-mRnOhaP|Rkjfr}BF-P*$X*-2LR#M0UAxrT%I+{t}%#-ZX+!RZ#Et~tE>}-(h zCFi>K@6QlSx!*8m*YurdF;nsjAvVZ(zqrHpUdT%)N0BiqO}bqE@hz`73hj+^4jps! zEoKI?4cpW_?=kOXOV&)u3Mg%^;TZxoIyRY5b!>!^ZBgl%tcC`Se&$5B-9U$oh>BLH zR#PpD-s~$KcMqOwi`=CmIcRc2H)!Xjnrm%TeXDI!P{(qsxZ+qVwGE2cG-}S~J(QYk zHmq54B&(tXpS}N&^B-~iKlzXNkNLk|WhIAfAOHd&00JNY0w4eaAOHd&00JQJ+!L6H zWg@+apZHj8+PP$tFtM(mJB1P*jZG(6P&T9Cj{iSo_Wys+e~3K*o_jq401yBH5C8!X z009sH0T2KI5CDNIOn?pWdp}r+hc9E7+5eC8Z*%a#~;ua z4iEqV5C8!X009sH0T2KI5C8!Xc!mkIqZ3n8*(tJSFquiD%hOh;(a0BHmTUW+_PW;C zvzv#)u5)F{b=5v(MhnOg0=Wp9Ep*!DKe})f&1z|ur7HetTiPP)03|jYcZ~pae|3M7 zh)|X}hC@Md&JT@_pcs^m!W_3n%D%Qo>-jBJSHzud zvC$%Tc5?T%)q88z&C2>6ac8%qH5H0e{+lb8u09M)ROm^>e*YikKjHXK=?e!4fB*=9 z00@8p2!H?xfB*=900@A<(@tO{HWhKFHloScRMMSt@W=n3@Si{J1%&({00JNY0w4ea zAOHd&00JNY0w4ea95Vyp|3o1=6nh~_KMZik{~vH{^*c|rTmhzTg%rLrNaE&VxgqWE%vOAABk3NPi@In z+V$PtfLzLnY`1g0(`@EMU2dtg`#7y#)^ytUoNZT5JD$r8Rqvj&?a|#`%>6ym_cdDP zye!(L)@Z0E#kCLHYA6FQS=uYzw8_6_=yk0jF4Jb~;xg@z?!~Nn3AJRsBR9okcdKh2NZY%ME3?jK?6P<~ zFmpnpDzpgP*%rQT8m%g$!Vplt&UJW?c-!a(@G>WdU;&ozxRA~bSNzdfB*=900@8p z2!H?xfB*=900>-J0yD7(ktdm!i{6YqNIuCNjywK;m*aoOzk6j%4Z%PF1V8`;KmY_l p00ck)1V8`;K;YRWP>g3bBfZ~PT=+6S{QHflL{XBLLrgr8( zZK3SYZJNZ53)daFO+wsu;esZlo!~Zc;Dj{61tbt>-t)7QIR8{>(^$(lYL(dU`ThC$ z`6WNcbw9ke()3K_W82x(J>@O#D97{MyNbeb+%)-}BEQiiOZZh?No4z}(O?@}DH2MAHTt>-0P2W#FNM9PYDb_;(0uX=z1pcoAdv9b@az4-B znep_SEz?_fOubQWnYwGbv4zR^S4&Hkl2TcEZ>6Nfl9ba~g*?J_&4!{>O4loN)z#JI z%S)>_luM->#UPKpZJ7>D>EOl@*r3+-Wx*)2)+_3d`k6>0)VBi85ph=O>f$ z|nVbYg1i|El%ypnUq|b=l3oKnVXHBy0LB=8*Q?&SKoF> zQEphm!1`?1_2u$~(sgCrra>RCl$C)Zl+zuZVsC1*QMmhdDkaa)^Y>2#dthMGffec4 zo)}0T+#4gYEq&8;+qz+n*&#jUlRX>l*l3@yo#yH8MxE~WG35}pFr7a2?^{-mWj9z; z#*|~M>y~A%e5vi0%p=^2`_ozpSt+fPL$>}8RTfI_yglupWP^P>i14_i36 zw1HI_GU?EdyxiW&WJOq`0Z{fZfeZ9?AvN^V7mM4wQJrg zd_Iwq&y(gA#x$>|UCDj4ONkvNBO60&-xXQy`JuHBs7#Elon;?ad$%KGNEdDitn!(m zm9vAoyM-%`t(;{aS2^wP)kIQ0bB4bWOb6jCN7nnMWte@7LRcM}l=~7-)1h3XCEojt zPsww0{1@-^O)+jy(-X0>dJ@73g%lJXbUjbeaol?}Ev;3qESE`XmrLbJI6<(tmCh6~ z_W171%H8av4s*vOlXthNbPl3fS*4C|btk2$&E4eAfgUB%=iwDu=8jjVlat$%cX(FU z_MLD(?dBbk)ic}OJmISdG(N$u>u#OAPnnM6x4ozl!?QZ3VLQ>(7ZpFe0W2i1Z<&tU zw5<^hP%e}{SXx`DC>m2W$Oi;(tT{Jz$7z~Qojz^pEm|}wbkp?ohVJRKVJ!F|y`ws$b1sQ+Xj^aJ z`AComEhoKH#;-hvHt+BBG(W)ABKa3i!mb$obfIt_>XuZU<(aPq64(Kq>eQG|}KqW_6 z=h!Ej2kh9V@joy9!I2+4AOHafKmY;|fB*y_009U<00Iy=90C$A%<Y33)dB%D>8Pp&GUjL8y|HZqa aLsXE;21w6D8F=v($sbA=M~116DRel7?M!zAFl0^e}Uz)mtefdA~o{=D4; z^ZK7Z&#e53N#ELMZte4*-KnmoSALfMN7_pL=k{ObMZ<6;fCP}hcZ9w3dw%2SaZz+dYvssL%9yIIk6ZL?WLR_M1 zeV5p(Ju-#4x}~`8u1#oeRA{pOEtc)LtVzyXPkES-Yh(@0xhfl1=ZQG3=8l}P=@}x{ z$UPke$)gsdGP@~g$Ha71DCQm!!Ls#Z-3$(F;@UcKLK2&TqnpPDQTv8|JaEI{tMH{w zp=)URPzBw@)?Dfi^Jx~|Pc3J6a_r?j-RzMwy?3U1-U!#gp)RXJ25kTw1*QYf-D^Hj z-WBG`3kOZ#t*N37sz~oa7JOxJwF<`aa`p|do1S5JEEbtfU1N4E9=AFUFQ`jP*-DN* zpJB{TTV!bY8biKl+=w4mc<0tq_WphLu~*+YPYmcvst$QFB4(FN9WKS{zNwQN)YUg3 zX5r3X*%excK6_Dr3x3 zl}_9j`nst@_2;?j_1a<%1qJ-_ZhAS}fU=^-FCU5Z3*>lud$2y+{1%IW$&#f?a~QBV^I(em51PR&0Xba+(F#X zdj;wt50rfcx{te!qus^>CDK(WCYG{Fj%7S=460Ubv>OrWOex0VIF~kN^@u0!RP}ydZ%Oci6k_+`UUa%iLRI(wAK7 zXlLh#KLM?&lqQi*r&FzoeW@ymI+v-*2n%p7VWg-bB;ivSEbO)9&(4Xc{ zpc9_2=V{M7znQ0lwQv-PL6G|!2xzxtdZ^H>*mNXF1(>rla75E=iwmc z(>xuwgjl9!5{T9vXr@UFKL#kY!o3cG;XKlZ$vrn;ckL0mkR`b)7A3JLx8zd2QmxD7 z*JY_vDOF^-d|#64k`#%QT2cv);S-&xh*C+EsN@BUV4FI=DO{|cuezjaJYW-4$4ljru;%X5&LI+VPI{F{( zu=4e)1lt9bq~CfJ<)-))<*l+LZEZ&wCK1*=gmICN?j-KEGnu1oDZBQXSBNNs*Djwx zDJpJPNwrq%iT!G?B0{cZ$Zn-3_NwJdtxM#Z)~{ZZ_mY=+>B1XBbw)6pZ|e>b+yN1E zvtyn4Kf!3XV-1PujdB0Ki%e0f$Z-8;6d5TjRdErRq9B?LQwnyJxvRx5@+R_zI*|YpKmter2_OL^fCP{L5zbZ2P@A)6CegZqVApsx@*tN-8Qzu-T8S+a@wKmter2_OL^ zfCP{L5CKmter z2_OL^fCP{L5$I>VIloM6h5YIpgejQ3E&Q?06MIxc+}q^LVv@ zH~+u-6~k|>e#QTte|mEnM9oM52_OL^fCP{L5); diff --git a/crates/tinyflows-module/src/catalog_documents.rs b/crates/tinyflows-module/src/catalog_documents.rs index f6dc86b3..0a58b12b 100644 --- a/crates/tinyflows-module/src/catalog_documents.rs +++ b/crates/tinyflows-module/src/catalog_documents.rs @@ -65,10 +65,9 @@ fn storage_error(error: CatalogStorageFailure) -> StorageError { "already_exists" => StorageError::already_exists(error.message), "not_found" => StorageError::not_found(error.message), "invalid_input" => StorageError::invalid_input(error.message), - "serialization" => StorageError::new( - tinyflows_storage::ErrorKind::Serialization, - error.message, - ), + "serialization" => { + StorageError::new(tinyflows_storage::ErrorKind::Serialization, error.message) + } "unavailable" => StorageError::unavailable(error.message), _ => StorageError::backend(error.message), } diff --git a/crates/tinyflows-module/src/catalog_migrations.rs b/crates/tinyflows-module/src/catalog_migrations.rs index ae89cedf..0658c649 100644 --- a/crates/tinyflows-module/src/catalog_migrations.rs +++ b/crates/tinyflows-module/src/catalog_migrations.rs @@ -14,10 +14,15 @@ pub(super) async fn migrate(connection: Connection, storage: CatalogStorage) -> documents.require_record_metadata().await?; let _guard = super::migration_lock::catalog().await; let documents = Arc::new(documents); - FlowCatalogDocuments::new(documents).ensure().await?; + FlowCatalogDocuments::new(documents.clone()) + .ensure() + .await?; let schema = catalog_storage_schema(); - let report = schema.migrate(&documents, &MigrateOptions::default()).await?; - let imported_kv = import_legacy_kv(&documents).await?; + let document_store: Arc = documents.clone(); + let report = schema + .migrate(&document_store, &MigrateOptions::default()) + .await?; + let imported_kv = import_legacy_kv(&document_store).await?; tracing::debug!( steps_run = report.ran.len(), already_done = report.already_done, diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index 09ca197a..bdd02937 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -1,5 +1,4 @@ use super::*; -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; use std::sync::{ @@ -11,9 +10,9 @@ use tinyflows_bus::catalog::{ PutWorkflowRequest, PutWorkflowResponse, }; use tinyflows_storage::{ - CollectionSpec, DocumentStore, Key, Precondition, Scope, StorageBackend, - WriteOptions, + CollectionSpec, DocumentStore, Key, Precondition, Scope, StorageBackend, WriteOptions, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; pub(super) struct HostDocuments { pub(super) documents: Arc, @@ -49,9 +48,7 @@ impl HostDocuments { && self.fail_revision_put.swap(false, Ordering::SeqCst) { return Ok(reply(Err::<(), _>( - tinyflows_storage::StorageError::unavailable( - "injected revision migration failure", - ), + tinyflows_storage::StorageError::unavailable("injected revision migration failure"), ))); } let mut options = WriteOptions::new(); diff --git a/crates/tinyflows-module/src/lib.rs b/crates/tinyflows-module/src/lib.rs index 4e6b828e..7ad74ddc 100644 --- a/crates/tinyflows-module/src/lib.rs +++ b/crates/tinyflows-module/src/lib.rs @@ -11,9 +11,9 @@ pub mod caps; mod catalog_authoring; mod catalog_documents; mod catalog_extended_ops; -mod catalog_legacy_import; mod catalog_legacy_batch; mod catalog_legacy_checkpoint; +mod catalog_legacy_import; mod catalog_migrations; mod catalog_service; mod catalog_service_summary; diff --git a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs index d1e57776..edc02b47 100644 --- a/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs +++ b/crates/tinyflows-module/src/run_manager/dedup_settlement_tests.rs @@ -6,7 +6,7 @@ use std::task::Poll; use tinyflows::caps::StateStore as _; use tinyflows_bus::catalog::{CallbackAddress, CatalogStorage}; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; fn storage(path: &str) -> CatalogStorage { CatalogStorage { @@ -33,7 +33,9 @@ fn graph_with_dedup() -> WorkflowGraph { } async fn catalog() -> FlowCatalogDocuments { - let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let backend = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); catalog.ensure().await.unwrap(); catalog diff --git a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs index 51ce25cd..ccfb1f42 100644 --- a/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs +++ b/crates/tinyflows-module/src/run_manager/events_persistence_test_support.rs @@ -1,4 +1,3 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use async_trait::async_trait; use serde_json::{Value, json}; use std::sync::{ @@ -10,9 +9,10 @@ use tinyflows_bus::catalog::{self, CatalogStorage}; use tinyflows_bus::runs::RunControlRequest; use tinyflows_drivers::catalog::FlowCatalogDocuments; use tinyflows_storage::{ - Capabilities, CollectionSpec, DocumentStore, Filter, MemoryStorage, Page, Precondition, Query, - Scope, Sort, StorageBackend, StorageError, Version, Versioned, WriteOptions, + Capabilities, CollectionSpec, DocumentStore, Filter, Page, Precondition, Query, Scope, Sort, + StorageBackend, StorageError, Version, Versioned, WriteOptions, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; pub(super) struct TestSetup { pub(super) catalog: FlowCatalogDocuments, @@ -23,7 +23,9 @@ pub(super) struct TestSetup { } pub(super) async fn setup(mode: FaultMode, status: &str) -> TestSetup { - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = Arc::new(FaultingDocuments { inner: scoped.documents().clone(), mode, @@ -173,10 +175,7 @@ impl DocumentStore for FaultingDocuments { self.inner.capabilities() } - async fn ensure_collection( - &self, - spec: &CollectionSpec, - ) -> tinyflows_storage::Result<()> { + async fn ensure_collection(&self, spec: &CollectionSpec) -> tinyflows_storage::Result<()> { self.inner.ensure_collection(spec).await } @@ -328,11 +327,7 @@ impl DocumentStore for FaultingDocuments { self.inner.query(collection, query).await } - async fn count( - &self, - collection: &str, - filter: &Filter, - ) -> tinyflows_storage::Result { + async fn count(&self, collection: &str, filter: &Filter) -> tinyflows_storage::Result { self.inner.count(collection, filter).await } diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index e6d708c1..b0d0fa04 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -13,7 +13,7 @@ use tinyflows_bus::runs::{ }; use tinyflows_catalog::FlowRunStep; use tinyflows_drivers::{DriverCheckpointer, catalog::FlowCatalogDocuments}; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; use tokio::sync::oneshot; #[tokio::test] @@ -249,7 +249,9 @@ struct RunningFixture { } async fn running_fixture() -> RunningFixture { - let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let backend = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = Arc::clone(backend.documents()); let catalog = FlowCatalogDocuments::new(Arc::clone(&docs)); let graph_json = json!({ diff --git a/crates/tinyflows-module/src/run_manager_tests.rs b/crates/tinyflows-module/src/run_manager_tests.rs index db4561c6..40a01379 100644 --- a/crates/tinyflows-module/src/run_manager_tests.rs +++ b/crates/tinyflows-module/src/run_manager_tests.rs @@ -1,8 +1,8 @@ use super::*; -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tinyflows_bus::catalog::CallbackAddress; use tinyflows_drivers::catalog::FlowCatalogDocuments; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; fn storage(owner: &str) -> tinyflows_bus::catalog::CatalogStorage { tinyflows_bus::catalog::CatalogStorage { @@ -15,7 +15,9 @@ fn storage(owner: &str) -> tinyflows_bus::catalog::CatalogStorage { #[tokio::test] async fn terminal_event_seals_cloned_progress_sinks() { - let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let backend = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); let run_id = format!("sealed-{}", uuid::Uuid::new_v4()); let graph = tinyflows::migrate::deserialize_graph(serde_json::json!({ @@ -96,7 +98,9 @@ fn duplicate_registration_keeps_the_original_cancellation_owner() { #[tokio::test] async fn admission_waiter_rechecks_after_rollback_and_after_success() { - let backend = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let backend = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let catalog = FlowCatalogDocuments::new(backend.documents().clone()); let run_id = format!("admission-{}", uuid::Uuid::new_v4()); let storage = storage("alice"); diff --git a/crates/tinyflows-module/src/schedule_clock_tests.rs b/crates/tinyflows-module/src/schedule_clock_tests.rs index 72fcd1ef..c49542b7 100644 --- a/crates/tinyflows-module/src/schedule_clock_tests.rs +++ b/crates/tinyflows-module/src/schedule_clock_tests.rs @@ -18,7 +18,7 @@ async fn retry_sleep_uses_tokio_time_without_changing_the_calendar_instant() { #[tokio::test] async fn a_manual_pass_uses_the_supplied_time_for_due_jobs_and_run_records() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host @@ -162,7 +162,7 @@ fn a_host_clock_reads_system_time_until_the_host_sets_an_instant() { #[tokio::test] async fn ticking_at_a_later_instant_runs_a_future_job_without_waiting_for_it() { use tinyflows_bus::schedule::{Schedule, StoreConfig}; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, _state) = tests::connections().await; let proxy = host diff --git a/crates/tinyflows-module/src/schedule_documents.rs b/crates/tinyflows-module/src/schedule_documents.rs index 415b59dd..8442bc5d 100644 --- a/crates/tinyflows-module/src/schedule_documents.rs +++ b/crates/tinyflows-module/src/schedule_documents.rs @@ -49,10 +49,9 @@ impl Failure { "already_exists" => StorageError::already_exists(self.message), "not_found" => StorageError::not_found(self.message), "invalid_input" => StorageError::invalid_input(self.message), - "serialization" => StorageError::new( - tinyflows_storage::ErrorKind::Serialization, - self.message, - ), + "serialization" => { + StorageError::new(tinyflows_storage::ErrorKind::Serialization, self.message) + } "unavailable" => StorageError::unavailable(self.message), _ => StorageError::backend(self.message), } diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index b9190274..0364358f 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -2,9 +2,7 @@ use super::*; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; -use tinyflows_storage::{ - Capability, ErrorKind, Scope, StorageBackend, WriteOp, -}; +use tinyflows_storage::{Capability, ErrorKind, Scope, StorageBackend, WriteOp}; struct Callback(Arc); #[tinybus::interface(name = "ai.tinyhumans.tinyflows.ScheduleDocuments")] @@ -37,7 +35,9 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( let connection = Connection::connect(bus.connect().await.unwrap()) .await .unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let storage = scoped.documents().clone(); storage .ensure_collection(&CollectionSpec::new("cron.jobs")) diff --git a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs index 9048e68b..991d29b0 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_lifecycle_service_tests.rs @@ -7,7 +7,7 @@ async fn prepared_manual_worker_stop_wait_and_ack_are_idempotent_and_fenced() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, StoreConfig, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -279,7 +279,7 @@ async fn delivery_projection_error_releases_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; state diff --git a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs index de53874b..d388c47a 100644 --- a/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_manual_worker_service_tests.rs @@ -19,7 +19,7 @@ async fn claim_reply_lost_after_commit_is_reconciled_and_released_before_termina ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -135,7 +135,7 @@ async fn release_reply_loss_reports_cleanup_pending_until_retry_confirms_fence() ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, StopManualRun, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; state @@ -274,7 +274,7 @@ async fn unsupported_claim_backend_fails_before_recording_claim_intent() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() @@ -388,7 +388,7 @@ async fn terminal_commit_error_releases_queued_claim_before_failed_terminal() { ManualDelivery, ManualRunOutcome, ManualRunRequest, ManualRunState, PrepareManualRun, StartManualRun, WaitManualRun, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; let scoped = MemoryStorage::new() diff --git a/crates/tinyflows-module/src/schedule_members_tests.rs b/crates/tinyflows-module/src/schedule_members_tests.rs index 34890634..52f7a963 100644 --- a/crates/tinyflows-module/src/schedule_members_tests.rs +++ b/crates/tinyflows-module/src/schedule_members_tests.rs @@ -1,5 +1,4 @@ -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; -//! Every contract-v3 pure member, called through the bus like a host would. +// Every contract-v3 pure member, called through the bus like a host would. use super::tests::{connections, document_callbacks}; use super::*; use serde_json::json; diff --git a/crates/tinyflows-module/src/schedule_service_tests.rs b/crates/tinyflows-module/src/schedule_service_tests.rs index a7193e2d..b1bb66e7 100644 --- a/crates/tinyflows-module/src/schedule_service_tests.rs +++ b/crates/tinyflows-module/src/schedule_service_tests.rs @@ -1,7 +1,7 @@ use super::*; -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use serde_json::json; use tinyflows_bus::schedule::{CALLBACK_INTERFACE, CallbackAddress, CronJob}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tokio::sync::Mutex; #[cfg(test)] @@ -126,7 +126,7 @@ async fn separate_schedule_object_parses_delays_and_rejects_cross_tenant_deliver #[tokio::test] async fn run_job_now_is_a_typed_module_owned_admission_operation() { use tinyflows_bus::schedule::{ManualDelivery, RunJobNow, RunJobReply, StoreConfig}; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, state) = connections().await; let backend = MemoryStorage::new(); @@ -223,7 +223,7 @@ async fn run_job_now_returns_typed_refusal_when_a_scheduled_claim_is_active() { use tinyflows_bus::schedule::{ ManualDelivery, RunJobNow, RunJobRefusal, RunJobReply, StoreConfig, }; - use tinyflows_storage::{ Scope, StorageBackend}; + use tinyflows_storage::{Scope, StorageBackend}; let (_server, host, _) = connections().await; let backend = MemoryStorage::new(); @@ -426,9 +426,11 @@ mod queue_tests; #[tokio::test] async fn module_document_store_retries_policy_migration_conflicts_without_losing_seeded_jobs() { - use tinyflows_storage::{ Precondition, Scope, StorageBackend}; + use tinyflows_storage::{Precondition, Scope, StorageBackend}; let (_server, host, _) = connections().await; - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = scoped.documents().clone(); host.serve_at( "/test/Documents".try_into().unwrap(), diff --git a/crates/tinyflows-module/src/schedule_store.rs b/crates/tinyflows-module/src/schedule_store.rs index 10b4d445..25062420 100644 --- a/crates/tinyflows-module/src/schedule_store.rs +++ b/crates/tinyflows-module/src/schedule_store.rs @@ -44,13 +44,17 @@ impl Store { Arc::new(super::schedule_documents::Documents::new(connection, callback).await?); documents.require_record_metadata()?; let _guard = super::migration_lock::schedule().await; - CronDocuments::new(documents) + CronDocuments::new(documents.clone()) .ensure() .await .map_err(anyhow::Error::msg)?; let schema = tinyflows_drivers::cron_storage_schema(); - let report = schema.migrate(&documents, &MigrateOptions::default()).await?; - let counter_imported = tinyflows_drivers::import_legacy_cron_counters(&documents).await?; + let document_store: Arc = documents.clone(); + let report = schema + .migrate(&document_store, &MigrateOptions::default()) + .await?; + let counter_imported = + tinyflows_drivers::import_legacy_cron_counters(&document_store).await?; Ok(serde_json::json!({ "steps_run": report.ran.len(), "already_done": report.already_done, diff --git a/crates/tinyflows-module/src/schedule_store_tests.rs b/crates/tinyflows-module/src/schedule_store_tests.rs index 805a085a..bb8d0571 100644 --- a/crates/tinyflows-module/src/schedule_store_tests.rs +++ b/crates/tinyflows-module/src/schedule_store_tests.rs @@ -1,11 +1,11 @@ use super::*; -use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use async_trait::async_trait; use chrono::{DateTime, Utc}; use std::time::Duration; use tinyflows_schedule::runtime::*; use tinyflows_schedule::{DeliveryStatus, Schedule}; -use tinyflows_storage::{ Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tokio::sync::Notify; struct Replacement(CronDocuments); diff --git a/crates/tinyflows-module/tests/fixtures/storage/README.md b/crates/tinyflows-module/tests/fixtures/storage/README.md new file mode 100644 index 00000000..26a1f1ad --- /dev/null +++ b/crates/tinyflows-module/tests/fixtures/storage/README.md @@ -0,0 +1,51 @@ +# Schedule storage compatibility fixtures + +These records were emitted by the public `CronDocuments::add_flow_schedule_job` +writer and then read back through the tagged release's `DocumentStore::get`. +They are retained to exercise the old-collection migration and the current +collection's read compatibility with the previous TinyFlows release. + +`cron-job-v0.8.8.json` came from TinyFlows tag `v0.8.8` +(`5882bf69993475b64ef6bcaaa5265fa389a5708d`). That release declared +`tinystoragedrivers-core` tag `v0.5.0`; the generated lockfile resolved it to +`fd96bea95a07abe06eb6039d2f1b8669ed0d2d09`. The public writer stored the job +in `cron_jobs` with its then-current serialized schedule/delivery fields. +Its fixture contains the returned id and document body. + +`cron-job-v0.8.28.json` came from TinyFlows tag `v0.8.28` +(`a587c7239b0726557b379d6840ab3603dbcfaa64`). That release pinned +`tinystoragedrivers-core` revision +`be94f7568fe7650b0729be55128f48270414b9a7` (TSD 1.0.0). Its public writer +stored the job in `cron.jobs`; the fixture records the returned document, +version, and metadata from the same backend read. + +Both generator sources are checked in beside this note. They were run from +archives of the exact TinyFlows tags with the declared TSD dependencies: + +```sh +mkdir -p target/legacy-v0.8.8 +git archive v0.8.8 | tar -x -C target/legacy-v0.8.8 +mkdir -p target/legacy-v0.8.8/crates/tinyflows-drivers/examples +cp crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.8.rs \ + target/legacy-v0.8.8/crates/tinyflows-drivers/examples/generate_cron_fixture.rs +cargo run --manifest-path target/legacy-v0.8.8/Cargo.toml -p tinyflows-drivers \ + --no-default-features --features schedule --example generate_cron_fixture \ + > crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.8.json + +mkdir -p target/legacy-v0.8.28 +git archive v0.8.28 | tar -x -C target/legacy-v0.8.28 +mkdir -p target/legacy-v0.8.28/crates/tinyflows-drivers/examples +cp crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.28.rs \ + target/legacy-v0.8.28/crates/tinyflows-drivers/examples/generate_cron_fixture.rs +cargo run --manifest-path target/legacy-v0.8.28/Cargo.toml -p tinyflows-drivers \ + --no-default-features --features schedule --example generate_cron_fixture \ + > crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.28.json +``` + +The v0.8.8 output is an envelope containing the legacy `cron_jobs` id and body. +The v0.8.28 output is an envelope containing the document and read-back +metadata. Their recorded SHA-256 digests are `4e7dacedea92c0d16348d45b1794699364b5ccae431d70320820c430845b4339` and +`ef28bcff4573ab326e007a128559801bc76f59b4ebbc352c057968f2272a827d`, +respectively. The writers generate current timestamps and UUIDs, so rerunning +produces different digest values. These are separate release artifacts; +neither combines old writer bytes with a newer engine. diff --git a/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.28.json b/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.28.json new file mode 100644 index 00000000..bb9d0798 --- /dev/null +++ b/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.28.json @@ -0,0 +1,31 @@ +{ + "doc": { + "claim_generation": 0, + "command": "base-fixture-flow", + "delete_after_run": false, + "delivery": { + "best_effort": true, + "channel": null, + "mode": "none", + "to": null + }, + "enabled": true, + "expression": "", + "incarnation": "494223b4-d8c9-4f59-83ea-99e63182ad61", + "job_type": "flow", + "name": "flow:base-fixture-flow", + "next_run_ms": 1792505373456, + "schedule": { + "every_ms": 60000, + "kind": "every" + }, + "session_target": "isolated" + }, + "id": "flow:base-fixture-flow", + "meta": { + "created_ms": 1792505313456, + "schema": 0, + "updated_ms": 1792505313456 + }, + "version": 1 +} diff --git a/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.8.json b/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.8.json new file mode 100644 index 00000000..9cd22178 --- /dev/null +++ b/crates/tinyflows-module/tests/fixtures/storage/cron-job-v0.8.8.json @@ -0,0 +1,20 @@ +{ + "doc": { + "command": "legacy-fixture-flow", + "created_at": "2026-10-20T14:15:31.602870071+00:00", + "created_ms": 1792505731602, + "delete_after_run": false, + "delivery": "{\"mode\":\"none\",\"channel\":null,\"to\":null,\"best_effort\":true}", + "enabled": true, + "expression": "", + "incarnation": "b6494c77-cc99-46e4-894b-17136325e4c3", + "job_type": "flow", + "name": "flow:legacy-fixture-flow", + "next_run": "2026-10-20T14:16:31.602870071+00:00", + "next_run_ms": 1792505791602, + "next_run_ns": 1792505791602870071, + "schedule": "{\"kind\":\"every\",\"every_ms\":60000}", + "session_target": "isolated" + }, + "id": "flow:legacy-fixture-flow" +} diff --git a/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.28.rs b/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.28.rs new file mode 100644 index 00000000..f7e8fd3d --- /dev/null +++ b/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.28.rs @@ -0,0 +1,29 @@ +use std::sync::Arc; + +use tinyflows_drivers::schedule::{CronDocuments, JOBS}; +use tinyflows_schedule::Schedule; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let backend = MemoryStorage::new(); + let docs = Arc::clone(backend.for_scope(&Scope::new("fixture")?)?.documents()); + let store = CronDocuments::new(Arc::clone(&docs)); + let job = store + .add_flow_schedule_job("base-fixture-flow", Schedule::Every { every_ms: 60_000 }) + .await?; + let record = docs + .get(JOBS, &job.id) + .await? + .expect("the public writer persisted the job"); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "id": record.id, + "version": record.version.0, + "doc": record.doc, + "meta": record.meta, + }))? + ); + Ok(()) +} diff --git a/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.8.rs b/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.8.rs new file mode 100644 index 00000000..67b44787 --- /dev/null +++ b/crates/tinyflows-module/tests/fixtures/storage/generators/generate_v0.8.8.rs @@ -0,0 +1,27 @@ +use std::sync::Arc; + +use tinyflows_drivers::schedule::{CronDocuments, JOBS}; +use tinyflows_schedule::Schedule; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let backend = MemoryStorage::new(); + let docs = Arc::clone(backend.for_scope(&Scope::new("fixture")?)?.documents()); + let store = CronDocuments::new(Arc::clone(&docs)); + let job = store + .add_flow_schedule_job("legacy-fixture-flow", Schedule::Every { every_ms: 60_000 }) + .await?; + let record = docs + .get(JOBS, &job.id) + .await? + .expect("the public writer persisted the job"); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "id": record.id, + "doc": record.doc, + }))? + ); + Ok(()) +} diff --git a/crates/tinyflows-module/tests/module_artifact_tests.rs b/crates/tinyflows-module/tests/module_artifact_tests.rs index fd3a8b6e..10ff80d3 100644 --- a/crates/tinyflows-module/tests/module_artifact_tests.rs +++ b/crates/tinyflows-module/tests/module_artifact_tests.rs @@ -359,8 +359,11 @@ async fn compiled_module_serves_validation_without_linking_engine_into_client() // Persist through the actual cdylib. The host forwards document I/O only; // schema, policy and schedule behavior is module-owned. - use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + use tinyflows_storage::{Scope, StorageBackend}; + use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = scoped.documents().clone(); let conflict_once = Arc::new(std::sync::atomic::AtomicBool::new(true)); let atomic_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); diff --git a/crates/tinyflows-module/tests/run_artifact/document_callback.rs b/crates/tinyflows-module/tests/run_artifact/document_callback.rs index 372a417a..6ecf0a6a 100644 --- a/crates/tinyflows-module/tests/run_artifact/document_callback.rs +++ b/crates/tinyflows-module/tests/run_artifact/document_callback.rs @@ -4,7 +4,7 @@ use std::sync::{ Arc, atomic::{AtomicBool, Ordering}, }; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; use tokio::sync::Notify; pub(crate) struct OwnerDocuments { @@ -211,7 +211,7 @@ fn decode(value: &Value) -> tinybus::Result { serde_json::from_value(value.clone()).map_err(tinybus::Error::from) } -fn reply(value: tinystoragedrivers_core::Result) -> Value { +fn reply(value: tinyflows_storage::Result) -> Value { match value { Ok(value) => json!({"value":serde_json::to_value(value).unwrap()}), Err(error) => json!({ diff --git a/crates/tinyflows-module/tests/run_artifact/support.rs b/crates/tinyflows-module/tests/run_artifact/support.rs index 24a2dab2..6f96f4e0 100644 --- a/crates/tinyflows-module/tests/run_artifact/support.rs +++ b/crates/tinyflows-module/tests/run_artifact/support.rs @@ -11,7 +11,8 @@ use tinyflows_bus::{ AgentRunHandle, AgentRunOutcome, AgentRunRequest, }, }; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use tokio::sync::Notify; #[path = "graphs.rs"] diff --git a/crates/tinyflows-module/tests/support/catalog_legacy_import_artifact_tests.rs b/crates/tinyflows-module/tests/support/catalog_legacy_import_artifact_tests.rs index dbf494a8..e1e3fce7 100644 --- a/crates/tinyflows-module/tests/support/catalog_legacy_import_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/catalog_legacy_import_artifact_tests.rs @@ -10,7 +10,8 @@ use tinyflows_bus::catalog::{ }; use tinyflows_bus::{BUS_NAME, catalog}; use tinyflows_drivers::DriverCheckpointer; -use tinystoragedrivers_core::{DocumentStore, MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{DocumentStore, Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use super::storage_migration_artifact_tests::CatalogCallback; diff --git a/crates/tinyflows-module/tests/support/document_callbacks.rs b/crates/tinyflows-module/tests/support/document_callbacks.rs index f36bbc92..89f4cd97 100644 --- a/crates/tinyflows-module/tests/support/document_callbacks.rs +++ b/crates/tinyflows-module/tests/support/document_callbacks.rs @@ -5,7 +5,7 @@ use std::sync::{ Arc, atomic::{AtomicBool, AtomicUsize, Ordering}, }; -use tinystoragedrivers_core::{DocumentStore, Result, StorageError}; +use tinyflows_storage::{DocumentStore, Result, StorageError}; fn decode(value: &Value) -> Result { Ok(serde_json::from_value(value.clone())?) } @@ -36,7 +36,7 @@ impl Documents { let id = request["id"].as_str().unwrap_or(""); match request["op"].as_str().unwrap_or("") { "capabilities" => Ok( - json!({"transactions":self.docs.capabilities().contains(tinystoragedrivers_core::Capability::Transactions),"record_metadata":true}), + json!({"transactions":self.docs.capabilities().contains(tinyflows_storage::Capability::Transactions),"record_metadata":true}), ), "atomic_batch" => { self.atomic_calls.fetch_add(1, Ordering::SeqCst); @@ -53,7 +53,7 @@ impl Documents { } "get" => encode(self.docs.get(collection, id).await?), "put" => { - let mut options = tinystoragedrivers_core::WriteOptions::new(); + let mut options = tinyflows_storage::WriteOptions::new(); if let Some(schema) = request["schema"].as_u64() { options = options.schema(u32::try_from(schema).unwrap_or(0)); } diff --git a/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs b/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs index bd537dd5..415fdd05 100644 --- a/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/manual_worker_artifact_tests.rs @@ -9,7 +9,8 @@ use tinyflows_bus::schedule::{ }; use tinyflows_drivers::schedule::CronDocuments; use tinyflows_schedule::{CronRun, Schedule}; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] #[ignore = "run with TINYFLOWS_MODULE_ARTIFACT set to a compiled module"] @@ -39,7 +40,9 @@ async fn compiled_module_serves_manual_worker_lifecycle_through_schedule_abi() { .await .unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let documents = CronDocuments::new(scoped.documents().clone()); let job = documents .add_shell_job( diff --git a/crates/tinyflows-module/tests/support/schedule_v4_artifact.rs b/crates/tinyflows-module/tests/support/schedule_v4_artifact.rs index 32ff73d1..0b9ef509 100644 --- a/crates/tinyflows-module/tests/support/schedule_v4_artifact.rs +++ b/crates/tinyflows-module/tests/support/schedule_v4_artifact.rs @@ -6,7 +6,7 @@ use serde_json::Value; use std::sync::Arc; use tinybus::Proxy; use tinyflows_bus::schedule::CallbackAddress; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; pub fn verify_captured_triggers(state: &callbacks::State, stored_job_id: &str) { let jobs = state.trigger_jobs.lock().unwrap(); diff --git a/crates/tinyflows-module/tests/support/schedule_v4_delivery_artifact.rs b/crates/tinyflows-module/tests/support/schedule_v4_delivery_artifact.rs index 1caac5a7..f0ccb18f 100644 --- a/crates/tinyflows-module/tests/support/schedule_v4_delivery_artifact.rs +++ b/crates/tinyflows-module/tests/support/schedule_v4_delivery_artifact.rs @@ -8,7 +8,7 @@ use tinyflows_bus::schedule::{ }; use tinyflows_drivers::schedule::CronDocuments; use tinyflows_schedule::{CronJobPatch, DeliveryStatus, Schedule}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; /// Verifies one actual send and truthful terminal projection through the cdylib. pub async fn verify( diff --git a/crates/tinyflows-module/tests/support/schedule_v4_manual_artifact.rs b/crates/tinyflows-module/tests/support/schedule_v4_manual_artifact.rs index 11c957f1..6fc648da 100644 --- a/crates/tinyflows-module/tests/support/schedule_v4_manual_artifact.rs +++ b/crates/tinyflows-module/tests/support/schedule_v4_manual_artifact.rs @@ -8,7 +8,7 @@ use tinyflows_bus::schedule::{ }; use tinyflows_drivers::schedule::CronDocuments; use tinyflows_schedule::{CronJobPatch, Schedule}; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; /// Exercises module-owned manual queueing and lease refusal through the loaded cdylib. pub async fn verify( diff --git a/crates/tinyflows-module/tests/support/schedule_v4_recovery_artifact.rs b/crates/tinyflows-module/tests/support/schedule_v4_recovery_artifact.rs index b4ac1807..1c1dcb7b 100644 --- a/crates/tinyflows-module/tests/support/schedule_v4_recovery_artifact.rs +++ b/crates/tinyflows-module/tests/support/schedule_v4_recovery_artifact.rs @@ -6,7 +6,7 @@ use tinybus::Proxy; use tinyflows_bus::schedule::{CallbackAddress, StartScheduler, StoreConfig}; use tinyflows_drivers::schedule::CronDocuments; use tinyflows_schedule::Schedule; -use tinystoragedrivers_core::DocumentStore; +use tinyflows_storage::DocumentStore; /// Recovers an expired persisted lease through the loaded cdylib, then proves /// the former owner cannot renew, release or complete the recovered run. diff --git a/crates/tinyflows-module/tests/support/scheduler_stop_artifact_tests.rs b/crates/tinyflows-module/tests/support/scheduler_stop_artifact_tests.rs index fa623968..f1aac04e 100644 --- a/crates/tinyflows-module/tests/support/scheduler_stop_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/scheduler_stop_artifact_tests.rs @@ -7,7 +7,8 @@ use tinybus::{ use tinyflows_bus::schedule::{CallbackAddress, StartScheduler}; use tinyflows_drivers::schedule::CronDocuments; use tinyflows_schedule::Schedule; -use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; +use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[tokio::test] #[ignore = "run with TINYFLOWS_MODULE_ARTIFACT set to a compiled cdylib"] diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 9c170b75..765ad07e 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -6,13 +6,14 @@ use tinyflows_bus::catalog::{ WorkflowRevision, }; use tinyflows_drivers::schedule::CronDocuments; -use tinystoragedrivers_core::{ - CollectionSpec, DocumentStore, MemoryStorage, Precondition, Scope, StorageBackend, WriteOptions, +use tinyflows_storage::{ + CollectionSpec, DocumentStore, Precondition, Scope, StorageBackend, WriteOptions, }; +use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; pub(super) struct CatalogCallback(pub(super) Arc); -fn catalog_reply(result: tinystoragedrivers_core::Result) -> Value { +fn catalog_reply(result: tinyflows_storage::Result) -> Value { match result { Ok(value) => json!({"value":serde_json::to_value(value).unwrap()}), Err(error) => { @@ -114,27 +115,20 @@ async fn published_module_migrates_legacy_schedule_records_through_owned_bus_cal broker.attach(Arc::new(transport)); let connection = Connection::connect(Box::new(client)).await.unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = scoped.documents().clone(); - docs.ensure_collection(&tinystoragedrivers_core::CollectionSpec::new("cron_jobs")) + docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("cron_jobs")) .await .unwrap(); + let legacy: Value = + serde_json::from_str(include_str!("../fixtures/storage/cron-job-v0.8.8.json")).unwrap(); + let legacy_id = legacy["id"].as_str().unwrap(); docs.put( "cron_jobs", - "legacy-job", - json!({ - "claim_generation": 1, - "expression": "0 9 * * *", - "command": "echo migrated", - "schedule": "{\"kind\":\"cron\",\"expr\":\"0 9 * * *\",\"tz\":null,\"active_hours\":null}", - "job_type": "shell", - "session_target": "isolated", - "enabled": true, - "delivery": "{\"mode\":\"none\",\"channel\":null,\"to\":null,\"best_effort\":true}", - "delete_after_run": false, - "created_at": "2026-03-04T05:06:07+00:00", - "next_run": "2026-03-05T09:00:00+00:00" - }), + legacy_id, + legacy["doc"].clone(), Precondition::None, ) .await @@ -181,11 +175,99 @@ async fn published_module_migrates_legacy_schedule_records_through_owned_bus_cal concurrent.unwrap(); let _: Value = schedule.call("ScheduleStore", &(request,)).await.unwrap(); - let migrated = docs.get("cron.jobs", "legacy-job").await.unwrap().unwrap(); - assert_eq!(migrated.meta.created_ms, 1_772_600_767_000_i64); + let migrated = docs.get("cron.jobs", legacy_id).await.unwrap().unwrap(); + let legacy_created_ms = + chrono::DateTime::parse_from_rfc3339(legacy["doc"]["created_at"].as_str().unwrap()) + .unwrap() + .timestamp_millis(); + assert_eq!(migrated.meta.created_ms, legacy_created_ms); let jobs = CronDocuments::new(docs).list_jobs().await.unwrap(); assert_eq!(jobs.len(), 1, "the legacy job is visible after migration"); - assert_eq!(jobs[0].id, "legacy-job"); + assert_eq!(jobs[0].id, legacy_id); + assert_eq!(jobs[0].command, "legacy-fixture-flow"); +} + +#[tokio::test] +#[ignore = "run through scripts/verify-module-artifact.sh"] +async fn published_module_reads_previous_release_schedule_records_on_open() { + let artifact = std::env::var_os("TINYFLOWS_MODULE_ARTIFACT") + .expect("set TINYFLOWS_MODULE_ARTIFACT to a compiled cdylib"); + let broker = Broker::new(); + let host = ModuleHost::new(broker.clone()); + host.load_file(std::path::PathBuf::from(artifact)).unwrap(); + let (client, transport) = MemoryTransport::pair(); + broker.attach(Arc::new(transport)); + let connection = Connection::connect(Box::new(client)).await.unwrap(); + + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); + let docs = scoped.documents().clone(); + docs.ensure_collection(&tinyflows_storage::CollectionSpec::new("cron.jobs")) + .await + .unwrap(); + let previous: Value = + serde_json::from_str(include_str!("../fixtures/storage/cron-job-v0.8.28.json")).unwrap(); + let previous_meta = &previous["meta"]; + let created_ms = previous_meta["created_ms"].as_i64().unwrap(); + let schema = u32::try_from(previous_meta["schema"].as_u64().unwrap()).unwrap(); + let previous_id = previous["id"].as_str().unwrap(); + let original = docs + .put_with( + "cron.jobs", + previous_id, + previous["doc"].clone(), + Precondition::None, + WriteOptions::new().created_ms(created_ms).schema(schema), + ) + .await + .unwrap(); + + connection + .serve_at( + "/test/ScheduleDocuments".try_into().unwrap(), + super::document_callbacks::Documents { + docs: docs.clone(), + conflict_once: Arc::new(AtomicBool::new(false)), + atomic_calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }, + ) + .await + .unwrap(); + connection + .request_name("test.ScheduleDocuments") + .await + .unwrap(); + let schedule = connection + .proxy( + tinyflows_bus::schedule::BUS_NAME, + tinyflows_bus::schedule::OBJECT_PATH, + tinyflows_bus::schedule::INTERFACE, + ) + .unwrap(); + let request = json!({ + "storage": { + "backend": {"kind":"documents", "callback": { + "destination":"test.ScheduleDocuments", + "object_path":"/test/ScheduleDocuments" + }} + }, + "operation": {"op":"migrate"} + }); + let _: Value = schedule.call("ScheduleStore", &(request,)).await.unwrap(); + + let reopened = docs + .get("cron.jobs", previous_id) + .await + .unwrap() + .expect("the previous release's schedule remains readable"); + assert_eq!(reopened.doc, previous["doc"]); + assert_eq!(reopened.version, original); + assert_eq!(reopened.meta.created_ms, created_ms); + let jobs = CronDocuments::new(docs).list_jobs().await.unwrap(); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0].id, previous_id); + assert_eq!(jobs[0].command, "base-fixture-flow"); } #[tokio::test] @@ -199,7 +281,9 @@ async fn published_module_migrates_legacy_catalog_records_through_owned_bus_call let (client, transport) = MemoryTransport::pair(); broker.attach(Arc::new(transport)); let connection = Connection::connect(Box::new(client)).await.unwrap(); - let scoped = MemoryStorage::new().for_scope(&Scope::new("local").expect("valid scope")).unwrap(); + let scoped = MemoryStorage::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .unwrap(); let docs = scoped.documents().clone(); for collection in [ "flows_definitions", @@ -314,8 +398,8 @@ async fn published_module_migrates_legacy_catalog_records_through_owned_bus_call .unwrap(); assert_eq!(migrated_revision.doc["created_ms"], revision_created_ms); assert_eq!(migrated_revision.meta.created_ms, revision_created_ms); - let catalog_state = tinystoragedrivers_core::Key::of(["flows", "kv:legacy-flow", "k"]).unwrap(); - let state = tinystoragedrivers_core::Key::of(["flows", "state", "state-key"]).unwrap(); + let catalog_state = tinyflows_storage::Key::of(["flows", "kv:legacy-flow", "k"]).unwrap(); + let state = tinyflows_storage::Key::of(["flows", "state", "state-key"]).unwrap(); assert_eq!( docs.get("tsd.kv", catalog_state.as_str()) .await diff --git a/crates/tinyflows-storage/src/blocking.rs b/crates/tinyflows-storage/src/blocking.rs index 62e56437..4aa39287 100644 --- a/crates/tinyflows-storage/src/blocking.rs +++ b/crates/tinyflows-storage/src/blocking.rs @@ -37,10 +37,11 @@ thread_local! { /// use tinyflows_storage::{Blocking, Precondition, Scope, StorageBackend, StorageError}; /// use tinystoragedrivers_testkit::MemoryBackend; /// use serde_json::json; +/// use std::sync::Arc; /// /// let bridge = Blocking::new()?; /// let storage = MemoryBackend::new().for_scope(&Scope::new("local")?)?; -/// let docs = storage.documents(); +/// let docs = Arc::clone(storage.documents()); /// let version = bridge.run(async move { /// docs.put("settings", "theme", json!({"dark": true}), Precondition::None).await /// })??; diff --git a/crates/tinyflows-storage/src/lib.rs b/crates/tinyflows-storage/src/lib.rs index 1ec1bde6..acbb4dc4 100644 --- a/crates/tinyflows-storage/src/lib.rs +++ b/crates/tinyflows-storage/src/lib.rs @@ -3,15 +3,22 @@ //! These helpers contain record and migration behavior owned by TinyFlows; //! backend selection stays with the host. -pub use tinystoragedrivers_api::{DocumentStore, DocumentStoreExt, Storage, StorageBackend, async_trait}; +pub use tinystoragedrivers_api::{ + DocumentStore, DocumentStoreExt, Storage, StorageBackend, async_trait, +}; pub use tinystoragedrivers_types::*; mod key; mod schema; pub use key::Key; -pub use schema::{MigrateOptions, MigrationReport, Migrations, Owner, Schema, ScopeClass, SourceRecord, Step, StepReport, TargetRecord}; +pub use schema::{ + MigrateOptions, MigrationReport, Migrations, Owner, Schema, ScopeClass, SourceRecord, Step, + StepReport, TargetRecord, +}; mod blocking; pub use blocking::Blocking; -pub use primitives::{Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy}; +pub use primitives::{ + Acquire, Acquisition, Counters, Kv, Lease, LeaseRecord, Leases, Repo, RetryPolicy, +}; pub(crate) type Clock = std::sync::Arc u64 + Send + Sync>; @@ -19,7 +26,9 @@ pub(crate) fn system_clock() -> u64 { use std::time::{SystemTime, UNIX_EPOCH}; SystemTime::now() .duration_since(UNIX_EPOCH) - .map_or(0, |duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }) } pub mod primitives { diff --git a/crates/tinyflows-storage/src/primitives/counters.rs b/crates/tinyflows-storage/src/primitives/counters.rs index cf5f4df5..644e328c 100644 --- a/crates/tinyflows-storage/src/primitives/counters.rs +++ b/crates/tinyflows-storage/src/primitives/counters.rs @@ -6,9 +6,9 @@ use serde::{Deserialize, Serialize}; use super::COUNTERS; use super::repo::{Repo, RetryPolicy}; +use crate::Key; use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; -use crate::Key; /// The largest value every driver can store (`i64::MAX`). const MAX_VALUE: u64 = i64::MAX as u64; diff --git a/crates/tinyflows-storage/src/primitives/kv.rs b/crates/tinyflows-storage/src/primitives/kv.rs index 1a3f7a9e..b2a22db9 100644 --- a/crates/tinyflows-storage/src/primitives/kv.rs +++ b/crates/tinyflows-storage/src/primitives/kv.rs @@ -8,9 +8,9 @@ use serde_json::Value; use super::KV; use super::repo::Repo; +use crate::Key; use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Filter, Precondition, Query, Result, Sort, StorageError, Version}; -use crate::Key; /// Entries per page while [`Kv::list`] walks a namespace. const PAGE: usize = 200; diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 5b601378..71bbaa15 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -17,10 +17,10 @@ use serde::{Deserialize, Serialize}; use super::LEASES; use super::repo::{Repo, RetryPolicy}; -use tinystoragedrivers_api::DocumentStore; -use tinystoragedrivers_types::{Result, StorageError}; use crate::Key; use crate::{Clock, system_clock}; +use tinystoragedrivers_api::DocumentStore; +use tinystoragedrivers_types::{Result, StorageError}; /// The stored body: `{holder, epoch, expires_ms}` plus the optional /// `endpoint` and `released` fields, both absent from records that never diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index 9d3e78ec..5ecca9ad 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -21,7 +21,7 @@ async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata( WriteOptions::new() .schema(1) .created_ms(123) - .expires_ms(999), + .expires_ms(i64::MAX), ) .await .expect("seed source"); @@ -55,7 +55,7 @@ async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata( assert_eq!(stored.doc["value"], "after"); assert_eq!(stored.meta.schema, 2); assert_eq!(stored.meta.created_ms, 123); - assert_eq!(stored.meta.expires_ms, Some(999)); + assert_eq!(stored.meta.expires_ms, Some(i64::MAX)); assert_eq!(first.ran.len(), 1); let second = schema @@ -137,6 +137,55 @@ async fn owner_migration_rejects_same_id_conflicts_in_a_different_declared_colle ); } +#[tokio::test] +async fn owner_migration_uses_the_declared_version_of_a_different_target_collection() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + docs.ensure_collection(&CollectionSpec::new("legacy.items")) + .await + .expect("legacy source collection"); + docs.put_with( + "legacy.items", + "one", + json!({"value": "before"}), + Precondition::Absent, + WriteOptions::new().schema(1), + ) + .await + .expect("seed source"); + + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .collection("flows.archive", 9, ScopeClass::Tenant) + .step( + Step::new("items_to_archive", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| { + TargetRecord::new("flows.archive", row.id, json!({"value": "archived"})) + .created_ms(456) + }) + .collect()) + }) + .source("legacy.items"), + ); + + schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("cross-collection migration succeeds"); + let archived = docs + .get("flows.archive", "one") + .await + .expect("read archived row") + .expect("archived row exists"); + assert_eq!(archived.meta.schema, 9); + assert_eq!(archived.meta.created_ms, 456); + assert_eq!(archived.doc["value"], "archived"); +} + #[tokio::test] async fn owner_migration_reads_compatible_shared_progress_records() { let storage = MemoryBackend::new() diff --git a/docs/storage-crate-split.md b/docs/storage-crate-split.md index db40362a..9b60a699 100644 --- a/docs/storage-crate-split.md +++ b/docs/storage-crate-split.md @@ -1,31 +1,28 @@ # Storage crate split (decision) -Status: decided 2026-10-13. Tracks tinyhumansai/openhuman#7754 and #7522. +Status: updated 2026-10-20. Tracks tinyhumansai/openhuman#7754, #7522 and #8131. ## Decision `tinyflows-drivers` is **not** folded into the published `tinyflows` crate. -The split stays until `tinystoragedrivers-core` is on crates.io. Nothing else -about tinyflows storage is open: the crates depend on the storage ports only -(`tinystoragedrivers-core`; `tinyflows-import` also takes the SQLite driver's -native handle to read legacy files). +The adapters depend on the stable public `tinystoragedrivers-api` and +`tinystoragedrivers-types` contracts through the internal `tinyflows-storage` +helper crate. `tinyflows-import` was retired after the v0.8.9 migration +transition; file and SQLite discovery is host-owned and no longer lives in +the TinyFlows runtime. ## Why -- `tinystoragedrivers-core` is `publish = false` and is not on crates.io. It is - named by git tag (`v0.7.0`). -- `cargo publish` rejects a manifest with a git-only dependency, optional or - not. The released `tinyflows` package therefore cannot name the ports. -- Moving the adapters into `tinyflows` behind a feature would not help: the - dependency is still declared in the published manifest. +- The published `tinyflows` engine remains separate from its optional driver + adapters, and does not gain a dependency on the private workspace helper. +- The helper crate keeps adapter-owned storage behavior separate while using + the published storage vocabulary. ## What would change this -Publishing `tinystoragedrivers-core` (a version-only dependency from -`tinyflows`). Then `tinyflows-drivers` becomes a `storage` feature of -`tinyflows`, `tinyflows-catalog` and `tinyflows-schedule` follow, and -`tinyflows-drivers` is deleted. That is a tinystoragedrivers release decision, -not a tinyflows change. +No further storage crate reshaping is planned. Hosts open and select the +backend, while TinyFlows receives scoped ports and declares only its owned +collections. ## Guard diff --git a/docs/tinyflows-bus-design.md b/docs/tinyflows-bus-design.md index 739e7b7e..72a09fa5 100644 --- a/docs/tinyflows-bus-design.md +++ b/docs/tinyflows-bus-design.md @@ -228,11 +228,11 @@ already removed it. The catalog and run checkpoints use the document port (`CatalogStorage`: ensure/get/put/delete/query/delete-where over scoped collections), backed on the host by `tinystoragedrivers` (sqlite, file, mongodb) selected by `OPENHUMAN_STORAGE_URL`. `tinyflows-drivers` keeps the document-port -implementation of catalog and checkpoint stores. `tinyflows-import` separates -host-side raw readers (`flow-read`, `checkpoint-read`, and `schedule-read`) -from semantic writers; the raw reader features do not link TinyFlows -implementation crates. No `tinyflows-sqlite` crate is created. Scope is the -agent id, set by the host when it binds the callback object. +implementation of catalog and checkpoint stores. Hosts own legacy source +discovery and raw readers; the module accepts bounded records through the +callback contract and applies TinyFlows-owned semantic writes. No +`tinyflows-sqlite` crate is created. Scope is the agent id, set by the host +when it binds the callback object. The served Catalog v2 receives a `CatalogStorage` callback address on each request. That object is created by the host for the acting agent, so the module diff --git a/scripts/check-legacy-import-modes.sh b/scripts/check-legacy-import-modes.sh deleted file mode 100755 index 556ad16b..00000000 --- a/scripts/check-legacy-import-modes.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -raw_features="flow-read,checkpoint-read,schedule-read" - -cargo test -p tinyflows-import --no-default-features --features "$raw_features" -cargo clippy -p tinyflows-import --no-default-features --features "$raw_features" --all-targets -- -D warnings -cargo test -p tinyflows-import --no-default-features --features catalog --test catalog_feature_contract_tests -cargo test -p tinyflows-import --no-default-features --features catalog - -raw_tree="$(cargo tree -p tinyflows-import --no-default-features --features "$raw_features" -e normal --prefix none --format '{p} features=[{f}]')" -for package in tinyflows tinyflows-catalog tinyflows-drivers tinyflows-schedule tinyflows-copilot; do - if grep -q "^${package} v" <<<"$raw_tree"; then - echo "raw reader mode unexpectedly links ${package}" >&2 - exit 1 - fi -done - -flow_write_tree="$(cargo tree -p tinyflows-import --no-default-features --features flow-write -e normal --prefix none --format '{p} features=[{f}]')" -if grep -Eq '^(tinystoragedrivers-sqlite|rusqlite) ' <<<"$flow_write_tree"; then - echo "flow-write unexpectedly links a SQLite reader" >&2 - exit 1 -fi diff --git a/scripts/check-storage-engine-free.sh b/scripts/check-storage-engine-free.sh index 8375bd46..39f04f13 100755 --- a/scripts/check-storage-engine-free.sh +++ b/scripts/check-storage-engine-free.sh @@ -3,7 +3,7 @@ set -euo pipefail cd "$(dirname "$0")/.." -engine_pattern='^(rusqlite|libsqlite3-sys|mongodb|bson|aws-sdk-s3|lapin|amq-protocol|sqlx|redb|sled|rocksdb|tinystoragedrivers-(document|kv|blob|queue|secrets|connect)|tinystoragedrivers)$' +engine_pattern='^(rusqlite|libsqlite3-sys|mongodb|bson|aws-sdk-s3|lapin|amq-protocol|sqlx|redb|sled|rocksdb|tinystoragedrivers-(core|document|kv|blob|queue|secrets|connect)|tinystoragedrivers)$' engine_names() { awk '{print $1}' <<<"$1" | grep -xE "$engine_pattern" || true } @@ -14,12 +14,16 @@ if [[ -z "$(engine_names $'tinystoragedrivers-document v2.0.0\n')" ]]; then echo "engine detector self-test failed to catch a backend crate" >&2 exit 1 fi +if [[ -z "$(engine_names $'tinystoragedrivers-core v0.5.0\n')" ]]; then + echo "engine detector self-test failed to catch the legacy storage core" >&2 + exit 1 +fi if [[ -n "$(engine_names $'tinystoragedrivers-api v2.0.0\ntinyflows-storage v0.1.0\n')" ]]; then echo "engine detector self-test rejected an API-only tree" >&2 exit 1 fi -packages=$(cargo metadata --no-deps --format-version 1 | python3 -c ' +packages=$(cargo metadata --locked --no-deps --format-version 1 | python3 -c ' import json, sys metadata = json.load(sys.stdin) library_kinds = {"lib", "rlib", "dylib", "cdylib", "staticlib", "proc-macro"} @@ -34,7 +38,7 @@ fi while IFS= read -r package; do [[ -n "$package" ]] || continue - tree=$(cargo tree -p "$package" --all-features -e normal,build --prefix none) + tree=$(cargo tree --locked -p "$package" --all-features -e normal,build --prefix none) engines=$(engine_names "$tree") if [[ -n "$engines" ]]; then printf 'storage implementation reached by %s:\n%s\n' "$package" "$engines" >&2 From b8998e0e894bcfc12555d60fd8edba62c7f4bbb6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 18:12:17 +0300 Subject: [PATCH 23/64] fix(storage): migrate adaptive state during open --- crates/tinyflows-adaptive/README.md | 2 +- crates/tinyflows-adaptive/docs/api.md | 2 +- .../tinyflows-adaptive/src/drivers/migrate.rs | 7 +- crates/tinyflows-adaptive/src/drivers/mod.rs | 5 +- .../src/drivers/vault_tests.rs | 4 +- crates/tinyflows-adaptive/src/storage.rs | 23 ++++- .../tinyflows-adaptive/src/storage_tests.rs | 68 ++++++++++++- .../tinyflows-catalog/src/build_registry.rs | 24 ++--- .../tinyflows-catalog/src/import/n8n/expr.rs | 36 +++---- .../src/import/n8n/node_mapping.rs | 8 +- .../tinyflows-copilot/src/tools/draft_edit.rs | 27 +++-- .../tinyflows-schedule/src/posix_weekday.rs | 2 +- crates/tinyflows-schedule/src/validation.rs | 6 +- crates/tinyflows-storage/src/schema.rs | 2 +- .../script_tests_part_01_tests.rs | 5 +- crates/tinyflows/src/migrate.rs | 8 +- .../tinyflows/src/nodes/integration/agent.rs | 99 +++++++++---------- .../src/nodes/integration/agent_prompt.rs | 8 +- .../src/nodes/integration/approval.rs | 38 ++++--- .../tinyflows/src/nodes/integration/schema.rs | 22 ++--- crates/tinyflows/src/preflight_inputs.rs | 56 +++++------ crates/tinyflows/src/validate/node_config.rs | 35 ++++--- .../tinyflows/tests/interception_e2e_tests.rs | 9 +- 23 files changed, 284 insertions(+), 212 deletions(-) diff --git a/crates/tinyflows-adaptive/README.md b/crates/tinyflows-adaptive/README.md index bea881b0..62ade210 100644 --- a/crates/tinyflows-adaptive/README.md +++ b/crates/tinyflows-adaptive/README.md @@ -301,7 +301,7 @@ backend (SQLite on a desktop, MongoDB in the cloud, files, memory), opened once and handed over as a document handle: ```rust -let storage = Storage::from_documents(scoped_storage.documents().clone()); +let storage = Storage::from_documents(scoped_storage.documents().clone()).await?; let tenant = storage.for_tenant(&user_id); // ledger and vault, scoped together ``` diff --git a/crates/tinyflows-adaptive/docs/api.md b/crates/tinyflows-adaptive/docs/api.md index 0be23f5c..2f4f1583 100644 --- a/crates/tinyflows-adaptive/docs/api.md +++ b/crates/tinyflows-adaptive/docs/api.md @@ -32,7 +32,7 @@ use tinyflows_adaptive::ledger::EpisodeStatus; use tinyflows_adaptive::workflows::Snapshot; // once, at boot -let storage = Storage::from_documents(scoped_storage.documents().clone()); // the host's backend +let storage = Storage::from_documents(scoped_storage.documents().clone()).await?; // migrate and open the host's backend let (ledger_root, vault_root) = (storage.ledger(), storage.vault()); let caps = Capabilities { llm: Arc::new(MyTieredClient::new(cfg)), /* … */ }; diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index 0137effd..f8c96b16 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -223,9 +223,10 @@ fn workflow(source: &SourceRecord) -> Option { /// `adaptive_counters_import`). New row and lesson ids keep counting from where /// v1 stopped. Returns how many sequences were carried. /// -/// `tsd.counters` belongs to `tsd`, so a document step cannot write it: the -/// host calls this once per scope after `storage_schema().migrate`. A completed -/// import is skipped; with nothing to carry it only records completion. +/// `tsd.counters` belongs to `tsd`, so a document step cannot write it. The +/// adaptive owner's `Storage::from_documents` open path calls this after its +/// schema migration. A completed import is skipped; with nothing to carry it +/// only records completion. /// /// # Errors /// diff --git a/crates/tinyflows-adaptive/src/drivers/mod.rs b/crates/tinyflows-adaptive/src/drivers/mod.rs index 169b9895..ebe8c65d 100644 --- a/crates/tinyflows-adaptive/src/drivers/mod.rs +++ b/crates/tinyflows-adaptive/src/drivers/mod.rs @@ -40,8 +40,9 @@ //! increment or share an id. First-write-wins edges (`$setOnInsert` in Mongo) //! are insert-only writes. //! -//! [`storage_schema`] declares the collections and their v1 -> v2 -//! steps; [`import_legacy_adaptive_counters`] carries the v1 sequences. +//! [`storage_schema`] declares the collections and their v1 -> v2 steps; +//! `Storage::from_documents` runs those steps and carries the v1 sequences +//! through [`import_legacy_adaptive_counters`] before returning the stores. mod convert; mod ledger; diff --git a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs index 954ff4e6..a25fec9b 100644 --- a/crates/tinyflows-adaptive/src/drivers/vault_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/vault_tests.rs @@ -46,7 +46,9 @@ async fn an_unparseable_record_is_an_error() { #[tokio::test] async fn storage_scopes_both_halves_together() { use crate::ledger::Ledger; - let storage = crate::storage::Storage::from_documents(memory_docs()); + let storage = crate::storage::Storage::from_documents(memory_docs()) + .await + .expect("open storage"); let tenant = storage.for_tenant("t"); assert_eq!(tenant.ledger().scope(), Some("t")); assert_eq!(tenant.vault().scope(), Some("t")); diff --git a/crates/tinyflows-adaptive/src/storage.rs b/crates/tinyflows-adaptive/src/storage.rs index 926d167a..93ed6449 100644 --- a/crates/tinyflows-adaptive/src/storage.rs +++ b/crates/tinyflows-adaptive/src/storage.rs @@ -45,6 +45,10 @@ pub enum StorageError { /// The vault backend refused to open. #[error("vault: {0}")] Vault(#[from] WorkflowError), + /// The host-provided document store could not be migrated or opened. + #[cfg(feature = "storage-drivers")] + #[error("document storage: {0}")] + DocumentStorage(#[from] tinyflows_storage::StorageError), } /// Where the storage setting is read from, when the environment supplies it. @@ -175,19 +179,28 @@ impl Storage { /// Both halves on a `tinystoragedrivers` document handle, in the global /// bucket; [`for_tenant`](Self::for_tenant) scopes them as for every - /// other backend. + /// other backend. This owner-open operation runs the `flows` schema + /// migrations and imports legacy adaptive counters before returning. /// /// The handle is usually one the host already bound to a storage scope /// (its agent, its user), so the adaptive tenants live inside it. + /// + /// # Errors + /// Returns an error if schema migration or legacy counter import fails. #[cfg(feature = "storage-drivers")] - #[must_use] - pub fn from_documents(docs: std::sync::Arc) -> Self { - Self { + pub async fn from_documents( + docs: std::sync::Arc, + ) -> Result { + crate::drivers::storage_schema() + .migrate(&docs, &tinyflows_storage::MigrateOptions::new()) + .await?; + crate::drivers::import_legacy_adaptive_counters(&docs).await?; + Ok(Self { ledger: AnyLedger::Driver(crate::drivers::DriverLedger::new(std::sync::Arc::clone( &docs, ))), vault: AnyVault::Driver(crate::drivers::DriverVault::new(docs)), - } + }) } } diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index 661b4c6b..cf8e3326 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -71,7 +71,7 @@ async fn one_document_handle_holds_both_halves_and_survives_reopening() { .expect("scope"); let docs = scoped.documents().clone(); - let storage = Storage::from_documents(docs.clone()); + let storage = Storage::from_documents(docs.clone()).await.expect("open"); let tenant = storage.for_tenant("user-7"); assert_eq!(tenant.ledger().scope(), Some("user-7")); assert_eq!(tenant.vault().scope(), Some("user-7")); @@ -87,10 +87,74 @@ async fn one_document_handle_holds_both_halves_and_survives_reopening() { .expect("put"); // A new `Storage` over the same handle sees both halves, still scoped. - let again = Storage::from_documents(docs); + let again = Storage::from_documents(docs).await.expect("reopen"); let tenant = again.for_tenant("user-7"); assert_eq!(tenant.ledger().rows("ep-1").await.expect("rows").len(), 1); assert_eq!(tenant.vault().load().await.expect("load").len(), 1); let other = again.for_tenant("user-8"); assert!(other.vault().load().await.expect("load").is_empty()); } + +#[cfg(feature = "storage-drivers")] +#[tokio::test] +async fn opening_documents_migrates_adaptive_records_and_imports_counters() { + use serde_json::json; + use tinyflows_storage::{DocumentStoreExt, Precondition, StorageBackend}; + + let scoped = MemoryStorage::new() + .for_scope(&tinyflows_storage::Scope::new("local").expect("scope")) + .expect("bind scope"); + let docs = scoped.documents().clone(); + docs.put( + "adaptive_rows", + "ldg_00000007", + json!({ + "episode": "episode-1", + "scope_key": "tenant-1", + "seq": 7, + "record": json!({ + "id": "ldg_00000007", + "episode": "episode-1", + "attempt": 1, + "approach_sig": "sig", + "approach_desc": "approach", + "workflow_id": "workflow-1", + "outcome": "completed", + "cause": "", + "cost_usd": 0.0, + "at": "2026-01-01T00:00:00Z", + "satisfied": true, + "advanced": false + }).to_string() + }), + Precondition::None, + ) + .await + .expect("seed legacy row"); + docs.put( + "adaptive_counters", + "adaptive_rows", + json!({ "seq": 9 }), + Precondition::None, + ) + .await + .expect("seed legacy sequence"); + + // Production open owns both schema migration and counter import. + let storage = Storage::from_documents(docs.clone()).await.expect("open"); + let tenant = storage.for_tenant("tenant-1"); + let migrated = tenant.ledger().rows("episode-1").await.expect("rows"); + assert_eq!(migrated.len(), 1); + assert_eq!(migrated[0].id, "ldg_00000007"); + assert_eq!(migrated[0].at, "2026-01-01T00:00:00Z"); + + let next = crate::ledger::conformance::row("episode-2", 10, "next"); + let id = tenant.ledger().append(&next).await.expect("append"); + assert_eq!(id, "ldg_00000010", "the old counter remains monotonic"); + assert!( + docs.query_all("adaptive_counters", &tinyflows_storage::Query::default()) + .await + .expect("old counters") + .is_empty() + ); +} diff --git a/crates/tinyflows-catalog/src/build_registry.rs b/crates/tinyflows-catalog/src/build_registry.rs index 43eaf34d..8b4a794a 100644 --- a/crates/tinyflows-catalog/src/build_registry.rs +++ b/crates/tinyflows-catalog/src/build_registry.rs @@ -96,18 +96,18 @@ pub fn cancel_build_turn_scoped(thread_id: &str, request_id: Option<&str>) -> bo ); return false; }; - if let Some(rid) = request_id { - if turn.request_id.as_deref() != Some(rid) { - tracing::debug!( - target: "flows", - thread_id = %thread_id, - request_id = %rid, - active_request_id = turn.request_id.as_deref().unwrap_or(""), - "[flows] build_registry: cancel ignored — request_id mismatch \ - (superseded/unrelated request)" - ); - return false; - } + if let Some(rid) = request_id + && turn.request_id.as_deref() != Some(rid) + { + tracing::debug!( + target: "flows", + thread_id = %thread_id, + request_id = %rid, + active_request_id = turn.request_id.as_deref().unwrap_or(""), + "[flows] build_registry: cancel ignored — request_id mismatch \ + (superseded/unrelated request)" + ); + return false; } turn.token.cancel(); tracing::info!( diff --git a/crates/tinyflows-catalog/src/import/n8n/expr.rs b/crates/tinyflows-catalog/src/import/n8n/expr.rs index b9dfdb04..12673127 100644 --- a/crates/tinyflows-catalog/src/import/n8n/expr.rs +++ b/crates/tinyflows-catalog/src/import/n8n/expr.rs @@ -54,24 +54,24 @@ pub(super) fn translate_expr(raw: &str, warnings: &mut Vec, n8n_name: &s }; // Trivial single-reference case: `$json.foo.bar` (or `$json["foo"]`). - if let Some(path) = inner.strip_prefix("$json") { - if let Some(jq_path) = json_path_to_jq(path) { - // `json_path_to_jq` returns the tail path RELATIVE to `$json` - // (`.foo`, `.foo.bar`, or `""` when there's no tail at all — see - // its doc). Prefixing `.item` here binds it at the tinyflows - // scope's `item` key instead of the scope root (see this fn's - // doc, R-C1): the empty-tail case naturally becomes `.item` (not - // `.item.`) because the tail is `""`, with no special-casing - // needed here. - tracing::debug!( - target: "flows", - node = %n8n_name, - %raw, - translated = %format!("=.item{jq_path}"), - "[flows] n8n_import: translated $json expression" - ); - return format!("=.item{jq_path}"); - } + if let Some(path) = inner.strip_prefix("$json") + && let Some(jq_path) = json_path_to_jq(path) + { + // `json_path_to_jq` returns the tail path RELATIVE to `$json` + // (`.foo`, `.foo.bar`, or `""` when there's no tail at all — see + // its doc). Prefixing `.item` here binds it at the tinyflows + // scope's `item` key instead of the scope root (see this fn's + // doc, R-C1): the empty-tail case naturally becomes `.item` (not + // `.item.`) because the tail is `""`, with no special-casing + // needed here. + tracing::debug!( + target: "flows", + node = %n8n_name, + %raw, + translated = %format!("=.item{jq_path}"), + "[flows] n8n_import: translated $json expression" + ); + return format!("=.item{jq_path}"); } warnings.push(untranslated_warning(n8n_name, raw)); diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs index caaf48cb..56ea2e95 100644 --- a/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs @@ -286,10 +286,10 @@ pub(super) fn map_http_request( _ => Map::new(), }; // Normalize the method key (`requestMethod` is the older n8n spelling). - if !cfg.contains_key("method") { - if let Some(method) = cfg.remove("requestMethod") { - cfg.insert("method".to_string(), method); - } + if !cfg.contains_key("method") + && let Some(method) = cfg.remove("requestMethod") + { + cfg.insert("method".to_string(), method); } if !cfg.contains_key("body") { if let Some(body) = cfg.remove("jsonBody") { diff --git a/crates/tinyflows-copilot/src/tools/draft_edit.rs b/crates/tinyflows-copilot/src/tools/draft_edit.rs index 783fbd38..9c10ae7b 100644 --- a/crates/tinyflows-copilot/src/tools/draft_edit.rs +++ b/crates/tinyflows-copilot/src/tools/draft_edit.rs @@ -411,24 +411,23 @@ impl Tool for EditWorkflowTool { // agent "Edits live on draft {id}", which would be false if the write // never landed, leaving the next turn silently iterating on a stale // draft. - if let Some(draft_id) = write_back_draft.as_deref() { - if let Err(e) = self + if let Some(draft_id) = write_back_draft.as_deref() + && let Err(e) = self .write_edit_to_draft(Some(draft_id), &edited, &name) .await - { - tracing::warn!( - target: "flows", - %name, - %draft_id, - error = %e, - "[flows] edit_workflow: draft write-back failed after validation passed" - ); - return Ok(ToolResult::error(format!( - "The edit passed validation, but could NOT be written back to draft \ + { + tracing::warn!( + target: "flows", + %name, + %draft_id, + error = %e, + "[flows] edit_workflow: draft write-back failed after validation passed" + ); + return Ok(ToolResult::error(format!( + "The edit passed validation, but could NOT be written back to draft \ {draft_id}: {e}\n\nThe draft still holds the PREVIOUS graph, not this edit. \ Retry edit_workflow." - ))); - } + ))); } // Full builder hard-gate stack + proposal payload (shared with revise). diff --git a/crates/tinyflows-schedule/src/posix_weekday.rs b/crates/tinyflows-schedule/src/posix_weekday.rs index 52d29627..064273b8 100644 --- a/crates/tinyflows-schedule/src/posix_weekday.rs +++ b/crates/tinyflows-schedule/src/posix_weekday.rs @@ -70,7 +70,7 @@ fn translate_item(item: &str) -> Result { // Starting on Sunday already covers it. return Ok(format!("{CRATE_SUNDAY}-7{suffix}")); } - let lands_on_sunday = (POSIX_SUNDAY_ALIAS - start) % step == 0; + let lands_on_sunday = (POSIX_SUNDAY_ALIAS - start).is_multiple_of(step); let mut parts = Vec::with_capacity(2); if start < POSIX_SUNDAY_ALIAS { parts.push(format!("{}-7{suffix}", start + 1)); diff --git a/crates/tinyflows-schedule/src/validation.rs b/crates/tinyflows-schedule/src/validation.rs index b13a4a9c..5b08d576 100644 --- a/crates/tinyflows-schedule/src/validation.rs +++ b/crates/tinyflows-schedule/src/validation.rs @@ -93,10 +93,10 @@ pub fn validate_delivery( reply_target, .. }) = origin + && channel.eq_ignore_ascii_case(original) + && to == reply_target { - if channel.eq_ignore_ascii_case(original) && to == reply_target { - return Ok(()); - } + return Ok(()); } if channel.eq_ignore_ascii_case("web") { return Ok(()); diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index bd41a825..0105e40c 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -251,7 +251,7 @@ impl Schema { .map(|row| SourceRecord { id: row.id.clone(), body: row.doc.clone(), - meta: row.meta.clone(), + meta: row.meta, }) .collect(); let outputs = (step.run)(inputs)?; diff --git a/crates/tinyflows/src/caps/host/script_tests/script_tests_part_01_tests.rs b/crates/tinyflows/src/caps/host/script_tests/script_tests_part_01_tests.rs index 44d4cde3..d7481934 100644 --- a/crates/tinyflows/src/caps/host/script_tests/script_tests_part_01_tests.rs +++ b/crates/tinyflows/src/caps/host/script_tests/script_tests_part_01_tests.rs @@ -157,11 +157,10 @@ async fn a_missing_interpreter_says_what_is_missing_rather_than_failing_opaquely // Only meaningful when python3 is genuinely absent; where it exists this // case cannot arise and the assertion is skipped. - if let Err(err) = err { - if err.to_string().contains("cannot run") { + if let Err(err) = err + && err.to_string().contains("cannot run") { assert!(err.to_string().contains("PATH"), "{err}"); } - } } // Unix-only: these run `ScriptLanguage::Shell`, which `run_script` refuses diff --git a/crates/tinyflows/src/migrate.rs b/crates/tinyflows/src/migrate.rs index 4c5f3fc7..6c1bc604 100644 --- a/crates/tinyflows/src/migrate.rs +++ b/crates/tinyflows/src/migrate.rs @@ -167,10 +167,10 @@ fn locate_graph_error(migrated: &Value, err: &serde_json::Error) -> String { let mut skeleton = migrated.clone(); if let Some(fields) = skeleton.as_object_mut() { for field in ELEMENT_ARRAYS { - if let Some(slot) = fields.get_mut(*field) { - if slot.is_array() { - *slot = Value::Array(Vec::new()); - } + if let Some(slot) = fields.get_mut(*field) + && slot.is_array() + { + *slot = Value::Array(Vec::new()); } } } diff --git a/crates/tinyflows/src/nodes/integration/agent.rs b/crates/tinyflows/src/nodes/integration/agent.rs index f0c7cdcc..0443c7ce 100644 --- a/crates/tinyflows/src/nodes/integration/agent.rs +++ b/crates/tinyflows/src/nodes/integration/agent.rs @@ -285,63 +285,58 @@ async fn run_turn_indexed( } else { value.get("tool_call").cloned() }; - if let Some(tool_call) = model_tool_call { - if let Some(slug) = tool_call.get("slug").and_then(Value::as_str) { - // Only a tool actually offered in `config.tools` may be invoked; - // keep the matched descriptor so its trusted `connection_ref` wins. - let offered_tool = cfg - .get("tools") - .and_then(Value::as_array) - .and_then(|tools| { - tools - .iter() - .find(|t| t.get("slug").and_then(Value::as_str) == Some(slug)) - }); - if let Some(offered_tool) = offered_tool { - tracing::debug!(slug, "agent tool sub-port: invoking model-elected tool"); - let args = tool_call.get("args").cloned().unwrap_or(Value::Null); - // Credentials come from trusted config only: the offered tool - // descriptor's `connection_ref`, else the node's. The model's - // `tool_call.connection_ref` is deliberately NOT trusted — a - // prompt-injection could otherwise elect an arbitrary host - // credential id for the call. - let tool_conn = offered_tool - .get("connection_ref") - .and_then(Value::as_str) - .or(conn); - let result = ctx.caps.tools.invoke(slug, args, tool_conn).await?; - if let Value::Object(map) = &mut value { - map.insert("tool_result".to_string(), result); - } - } else { - tracing::warn!( - slug, - "agent tool sub-port: model elected an un-offered tool; ignoring" - ); + if let Some(tool_call) = model_tool_call + && let Some(slug) = tool_call.get("slug").and_then(Value::as_str) + { + // Only a tool actually offered in `config.tools` may be invoked; + // keep the matched descriptor so its trusted `connection_ref` wins. + let offered_tool = cfg + .get("tools") + .and_then(Value::as_array) + .and_then(|tools| { + tools + .iter() + .find(|t| t.get("slug").and_then(Value::as_str) == Some(slug)) + }); + if let Some(offered_tool) = offered_tool { + tracing::debug!(slug, "agent tool sub-port: invoking model-elected tool"); + let args = tool_call.get("args").cloned().unwrap_or(Value::Null); + // Credentials come from trusted config only: the offered tool + // descriptor's `connection_ref`, else the node's. The model's + // `tool_call.connection_ref` is deliberately NOT trusted — a + // prompt-injection could otherwise elect an arbitrary host + // credential id for the call. + let tool_conn = offered_tool + .get("connection_ref") + .and_then(Value::as_str) + .or(conn); + let result = ctx.caps.tools.invoke(slug, args, tool_conn).await?; + if let Value::Object(map) = &mut value { + map.insert("tool_result".to_string(), result); } + } else { + tracing::warn!( + slug, + "agent tool sub-port: model elected an un-offered tool; ignoring" + ); } } // Output-parser sub-port: validate/repair the agent output against a schema. - if let Some(parser) = cfg.get("output_parser").filter(|p| !p.is_null()) { - if let Some(parser_schema) = parser.get("schema").filter(|s| !s.is_null()) { - let auto_fix = parser - .get("auto_fix") - .and_then(Value::as_bool) - .unwrap_or(true); - let parser_conn = parser - .get("connection_ref") - .and_then(Value::as_str) - .or(conn); - value = schema::parse_and_validate( - value, - parser_schema, - auto_fix, - &ctx.caps.llm, - parser_conn, - ) - .await?; - } + if let Some(parser) = cfg.get("output_parser").filter(|p| !p.is_null()) + && let Some(parser_schema) = parser.get("schema").filter(|s| !s.is_null()) + { + let auto_fix = parser + .get("auto_fix") + .and_then(Value::as_bool) + .unwrap_or(true); + let parser_conn = parser + .get("connection_ref") + .and_then(Value::as_str) + .or(conn); + value = + schema::parse_and_validate(value, parser_schema, auto_fix, &ctx.caps.llm, parser_conn) + .await?; } // Emit the stable envelope: `json` is the structured (tool/parser-threaded) diff --git a/crates/tinyflows/src/nodes/integration/agent_prompt.rs b/crates/tinyflows/src/nodes/integration/agent_prompt.rs index 29f30f10..53be5919 100644 --- a/crates/tinyflows/src/nodes/integration/agent_prompt.rs +++ b/crates/tinyflows/src/nodes/integration/agent_prompt.rs @@ -278,10 +278,10 @@ pub fn extract_balanced_json(text: &str) -> Option { if depth == 0 { // Found a balanced span — try to parse it. let candidate = &text[start..=end]; - if let Ok(parsed) = serde_json::from_str::(candidate) { - if matches!(parsed, Value::Object(_) | Value::Array(_)) { - return Some(parsed); - } + if let Ok(parsed) = serde_json::from_str::(candidate) + && matches!(parsed, Value::Object(_) | Value::Array(_)) + { + return Some(parsed); } // Span didn't parse; continue scanning from the position // after this false-positive open byte. diff --git a/crates/tinyflows/src/nodes/integration/approval.rs b/crates/tinyflows/src/nodes/integration/approval.rs index e1f1f042..12a40647 100644 --- a/crates/tinyflows/src/nodes/integration/approval.rs +++ b/crates/tinyflows/src/nodes/integration/approval.rs @@ -171,18 +171,17 @@ impl NodeExecutor for ApprovalNode { // card any more, so withdraw it rather than leave a stale entry // in the host's queue. Best-effort: the run has already decided // what to do, and a failed withdrawal must not change that. - if let Some(provider) = ctx.caps.approvals.as_ref() { - if let Err(err) = provider + if let Some(provider) = ctx.caps.approvals.as_ref() + && let Err(err) = provider .cancel(&request.request_id, "resolved via resume") .await - { - tracing::warn!( - node = %ctx.node.id, - request = %request.request_id, - error = %err, - "withdrawing the provider's review after a resume decision failed" - ); - } + { + tracing::warn!( + node = %ctx.node.id, + request = %request.request_id, + error = %err, + "withdrawing the provider's review after a resume decision failed" + ); } ApprovalOutcome::Decided(decision) } @@ -293,18 +292,17 @@ impl ApprovalNode { // any more, so withdraw it rather than leaving a dead card in a queue. // Best-effort: the run has already decided what to do, and a failed // withdrawal must not change that. - if let Some(provider) = ctx.caps.approvals.as_ref() { - if let Err(err) = provider + if let Some(provider) = ctx.caps.approvals.as_ref() + && let Err(err) = provider .cancel(&request.request_id, "approval node timed out") .await - { - tracing::warn!( - node = %ctx.node.id, - request = %request.request_id, - error = %err, - "withdrawing the timed-out review failed" - ); - } + { + tracing::warn!( + node = %ctx.node.id, + request = %request.request_id, + error = %err, + "withdrawing the timed-out review failed" + ); } // The `=nodes..decision.approved` contract other settled reviews diff --git a/crates/tinyflows/src/nodes/integration/schema.rs b/crates/tinyflows/src/nodes/integration/schema.rs index 0c0c9841..4e639e00 100644 --- a/crates/tinyflows/src/nodes/integration/schema.rs +++ b/crates/tinyflows/src/nodes/integration/schema.rs @@ -43,12 +43,12 @@ fn validate_at(path: &str, value: &Value, schema: &Value, errors: &mut Vec Vec<(String, Vec)> { while i < bytes.len() { let nested_dot = i > 0 && (bytes[i - 1].is_ascii_alphanumeric() || matches!(bytes[i - 1], b'_' | b']')); - if bytes[i] == b'.' && !nested_dot { - if let Some((end, scope)) = root_selector(body, i + 1) { - match scope { - Some(scope) - if matches!( - scope.as_str(), - "inputs" | "run" | "item" | "items" | "nodes" - ) => - { - let path = - input_path(&format!("=inputs{}", &body[end..])).unwrap_or_default(); - reads.push((scope, path)); - } - None => reads.push(("run".into(), vec![])), - _ => {} + if bytes[i] == b'.' + && !nested_dot + && let Some((end, scope)) = root_selector(body, i + 1) + { + match scope { + Some(scope) + if matches!( + scope.as_str(), + "inputs" | "run" | "item" | "items" | "nodes" + ) => + { + let path = input_path(&format!("=inputs{}", &body[end..])).unwrap_or_default(); + reads.push((scope, path)); } - i = end; - continue; + None => reads.push(("run".into(), vec![])), + _ => {} } + i = end; + continue; } if bytes[i] == b'"' || bytes[i] == b'\'' { let quote = bytes[i]; @@ -229,17 +229,17 @@ pub(super) fn scope_reads(expression: &str) -> Vec<(String, Vec)> { i += 1; } } - if root_access && i < bytes.len() && bytes[i] == b']' { - if let Ok(scope) = serde_json::from_str::(&body[start..i]) { - if matches!( - scope.as_str(), - "inputs" | "run" | "item" | "items" | "nodes" - ) { - let path = - input_path(&format!("=inputs{}", &body[i + 1..])).unwrap_or_default(); - reads.push((scope, path)); - } - } + if root_access + && i < bytes.len() + && bytes[i] == b']' + && let Ok(scope) = serde_json::from_str::(&body[start..i]) + && matches!( + scope.as_str(), + "inputs" | "run" | "item" | "items" | "nodes" + ) + { + let path = input_path(&format!("=inputs{}", &body[i + 1..])).unwrap_or_default(); + reads.push((scope, path)); } } else if bytes[i].is_ascii_alphabetic() || bytes[i] == b'_' { let start = i; diff --git a/crates/tinyflows/src/validate/node_config.rs b/crates/tinyflows/src/validate/node_config.rs index c94a2c56..27d06bd7 100644 --- a/crates/tinyflows/src/validate/node_config.rs +++ b/crates/tinyflows/src/validate/node_config.rs @@ -207,16 +207,16 @@ pub(super) fn validate_node_configs(graph: &WorkflowGraph, errors: &mut Vec) -> StepAction { - if frame.phase == self.phase && frame.node.id == self.node_id { - if let Some(action) = self.action.lock().expect("action lock").take() { - return action; - } + if frame.phase == self.phase + && frame.node.id == self.node_id + && let Some(action) = self.action.lock().expect("action lock").take() + { + return action; } StepAction::Continue { state_patch: None } } From 3c20ebc0d5a134653fb896f11b7ea67e2f745cce Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 18:24:13 +0300 Subject: [PATCH 24/64] fix(msrv): pin UUID to Rust 1.88-compatible release --- Cargo.lock | 6 +++--- crates/tinyflows-adaptive/src/storage_tests.rs | 1 + crates/tinyflows-adaptive/tests/support/driver_store.rs | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c95f392..b7363477 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1990,7 +1990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2660,9 +2660,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.28.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", diff --git a/crates/tinyflows-adaptive/src/storage_tests.rs b/crates/tinyflows-adaptive/src/storage_tests.rs index cf8e3326..d796d747 100644 --- a/crates/tinyflows-adaptive/src/storage_tests.rs +++ b/crates/tinyflows-adaptive/src/storage_tests.rs @@ -1,4 +1,5 @@ use super::*; +#[cfg(feature = "storage-drivers")] use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; #[test] diff --git a/crates/tinyflows-adaptive/tests/support/driver_store.rs b/crates/tinyflows-adaptive/tests/support/driver_store.rs index e2eb1843..4897d4f1 100644 --- a/crates/tinyflows-adaptive/tests/support/driver_store.rs +++ b/crates/tinyflows-adaptive/tests/support/driver_store.rs @@ -11,7 +11,8 @@ use std::sync::Arc; use tinyflows::store::WorkflowStore; use tinyflows_drivers::store::DriverWorkflowStore; -use tinyflows_storage::{Scope, StorageBackend}; +use tinystoragedrivers_api::StorageBackend; +use tinystoragedrivers_types::Scope; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; /// An empty store over its own in-memory backend. From 4a3bf9eb2dedcc607505162eed90db4d355b8a47 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 18:30:24 +0300 Subject: [PATCH 25/64] style(tests): order storage helper imports --- crates/tinyflows-adaptive/tests/support/driver_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyflows-adaptive/tests/support/driver_store.rs b/crates/tinyflows-adaptive/tests/support/driver_store.rs index 4897d4f1..85757bf5 100644 --- a/crates/tinyflows-adaptive/tests/support/driver_store.rs +++ b/crates/tinyflows-adaptive/tests/support/driver_store.rs @@ -12,8 +12,8 @@ use std::sync::Arc; use tinyflows::store::WorkflowStore; use tinyflows_drivers::store::DriverWorkflowStore; use tinystoragedrivers_api::StorageBackend; -use tinystoragedrivers_types::Scope; use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; +use tinystoragedrivers_types::Scope; /// An empty store over its own in-memory backend. pub fn empty() -> DriverWorkflowStore { From b28e10e3e7c1dd75e31f64556465cbc9fe14c19f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 18:39:58 +0300 Subject: [PATCH 26/64] fix(storage): avoid unused unit lint --- crates/tinyflows-storage/src/blocking_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyflows-storage/src/blocking_tests.rs b/crates/tinyflows-storage/src/blocking_tests.rs index d0467b19..664ddbb7 100644 --- a/crates/tinyflows-storage/src/blocking_tests.rs +++ b/crates/tinyflows-storage/src/blocking_tests.rs @@ -31,7 +31,7 @@ fn a_future_cannot_reenter_its_own_bridge() { let bridge = Blocking::new().expect("bridge starts"); let inner = bridge.clone(); let error = bridge - .run(async move { inner.run(async { () }) }) + .run(async move { inner.run(async {}) }) .expect("outer bridge call completes") .expect_err("reentry would deadlock"); assert_eq!(error.kind(), tinystoragedrivers_types::ErrorKind::Backend); From f1bd34efb4e0c80156fddb9f934a585b23d86dc2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:29:36 +0300 Subject: [PATCH 27/64] chore(deps): bump tinystoragedrivers to 2.0.1 and add busproxy Updates the tinystoragedrivers api, types, and testkit dependencies to 2.0.1. Adds the busproxy testkit crate and the tinybus SDK as new dev-dependencies to support bus-backed storage tests. Auto-committed-on: dragonfly --- crates/tinyflows-storage/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index 62449f46..544bb57f 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -9,8 +9,8 @@ publish = false description = "TinyFlows-owned generic document persistence helpers" [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } serde = { workspace = true } serde_json = { workspace = true } async-trait = { workspace = true } @@ -20,4 +20,6 @@ ulid = "1" data-encoding = "2" [dev-dependencies] -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinybus-sdk = { package = "tinybus", git = "https://github.com/senamakel/tinybus", rev = "c7560289fd6f58b57622234f6d72ff9545cb38c9", default-features = false, features = ["modules"] } From 06e6971b0128e1c5443466db0a96af4d34d6d912 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:30:27 +0300 Subject: [PATCH 28/64] chore(deps): bump tinystoragedrivers to v2.0.1 Update the tinystoragedrivers api, types and testkit dependencies to v2.0.1 across the adaptive, drivers and module crates. Add a storage schema test that runs a migration through the real TSD busproxy to confirm the module-compatible path avoids sending host-only metadata. Auto-committed-on: dragonfly --- crates/tinyflows-adaptive/Cargo.toml | 10 +- crates/tinyflows-drivers/Cargo.toml | 6 +- crates/tinyflows-module/Cargo.toml | 6 +- crates/tinyflows-storage/src/schema_tests.rs | 99 ++++++++++++++++++++ 4 files changed, 110 insertions(+), 11 deletions(-) diff --git a/crates/tinyflows-adaptive/Cargo.toml b/crates/tinyflows-adaptive/Cargo.toml index 4cc6ccee..2e3cb57e 100644 --- a/crates/tinyflows-adaptive/Cargo.toml +++ b/crates/tinyflows-adaptive/Cargo.toml @@ -19,8 +19,8 @@ thiserror = "2" # tag so a host that vendors tinystoragedrivers once can `[patch]` it; chrono # turns the RFC 3339 instants of the records into the stored `*_ms`; tokio's # `OnceCell` for one-time collection setup. -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0", optional = true } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0", optional = true } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1", optional = true } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1", optional = true } tinyflows-storage = { path = "../tinyflows-storage", optional = true } chrono = { version = "0.4", default-features = false, features = ["std"], optional = true } tracing = { version = "0.1", default-features = false, features = ["std"], optional = true } @@ -41,9 +41,9 @@ tinyflows = { path = "../tinyflows", features = ["store", "mock"] } # The engine's `WorkflowStore` on an in-memory storage backend, for tests and # the eval example. tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["store"] } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [lints.rust] diff --git a/crates/tinyflows-drivers/Cargo.toml b/crates/tinyflows-drivers/Cargo.toml index 19ad64de..5cb48bf7 100644 --- a/crates/tinyflows-drivers/Cargo.toml +++ b/crates/tinyflows-drivers/Cargo.toml @@ -51,8 +51,8 @@ store = [ test-fixtures = ["catalog"] [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } tinyflows-storage = { path = "../tinyflows-storage" } serde = { workspace = true } serde_json = { workspace = true } @@ -72,4 +72,4 @@ uuid = { workspace = true, optional = true } [dev-dependencies] async-trait = { workspace = true } tokio = { workspace = true } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } diff --git a/crates/tinyflows-module/Cargo.toml b/crates/tinyflows-module/Cargo.toml index 5d644531..6ef4aad2 100644 --- a/crates/tinyflows-module/Cargo.toml +++ b/crates/tinyflows-module/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["rlib", "cdylib"] [dependencies] tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["catalog", "engine", "schedule"] } tinyflows-catalog = { path = "../tinyflows-catalog" } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } tinyflows-storage = { path = "../tinyflows-storage" } tinyflows-schedule = { path = "../tinyflows-schedule" } anyhow = "1" @@ -42,4 +42,4 @@ static-link = [] [dev-dependencies] tempfile = "3" tokio = { workspace = true, features = ["test-util"] } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.0", tag = "v2.0.0" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index 5ecca9ad..8788c851 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -3,6 +3,105 @@ use super::*; use serde_json::json; use std::sync::Arc; use tinystoragedrivers_testkit::MemoryBackend; +use tinystoragedrivers_busproxy::{PerCallerEndpoint, StorageServer, connect_storage, serve}; +use tinybus_sdk::{Connection, broker::Broker, transport::memory::MemoryBus}; + +struct BusDocumentsFixture { + _broker_task: tokio::task::JoinHandle>, + _host: Connection, + _served: tinystoragedrivers_busproxy::ServedStorage, + backing: tinystoragedrivers_api::Storage, + client: tinystoragedrivers_api::Storage, +} + +impl BusDocumentsFixture { + async fn new() -> Self { + let bus = MemoryBus::new(); + let broker = Broker::new(); + let broker_task = broker.spawn(bus.clone()); + let host = Connection::connect(bus.connect().await.expect("host transport")) + .await + .expect("host connection"); + let backing = MemoryBackend::new() + .for_scope(&Scope::new("schema-bus").expect("valid test scope")) + .expect("memory storage"); + let endpoint_storage = backing.clone(); + let endpoint = PerCallerEndpoint::new(move |_| { + Some(StorageServer::new(endpoint_storage.clone())) + }); + let served = serve(&host, Arc::new(endpoint)) + .await + .expect("serve TSD storage endpoint"); + let client_connection = Connection::connect(bus.connect().await.expect("client transport")) + .await + .expect("client connection"); + let client = connect_storage(client_connection) + .await + .expect("connect through TSD busproxy"); + Self { + _broker_task: broker_task, + _host: host, + _served: served, + backing, + client, + } + } +} + +#[tokio::test] +async fn schema_migration_uses_metadata_supported_by_the_real_tsd_busproxy() { + let fixture = BusDocumentsFixture::new().await; + let backing_docs = Arc::clone(fixture.backing.documents()); + backing_docs + .ensure_collection(&CollectionSpec::new("legacy.items")) + .await + .expect("legacy collection"); + backing_docs + .put_with( + "legacy.items", + "one", + json!({"value":"before"}), + Precondition::Absent, + WriteOptions::new().schema(1).expires_ms(i64::MAX), + ) + .await + .expect("seed legacy record"); + + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .step( + Step::new("items_v2", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| { + TargetRecord::new("flows.items", row.id, json!({"value":"after"})) + .created_ms(456) + }) + .collect()) + }) + .source("legacy.items"), + ); + let client_docs = Arc::clone(fixture.client.documents()); + schema + .migrate(&client_docs, &MigrateOptions::default()) + .await + .expect("module-compatible migration must not send host-only created_ms"); + + let migrated = backing_docs + .get("flows.items", "one") + .await + .expect("read migrated record") + .expect("record migrated"); + assert_eq!(migrated.doc, json!({"value":"after"})); + assert_eq!(migrated.meta.schema, 2); + assert_eq!(migrated.meta.expires_ms, Some(i64::MAX)); + assert!(migrated.meta.created_ms > 0); + assert!(backing_docs + .get("legacy.items", "one") + .await + .expect("read old source") + .is_none()); +} #[tokio::test] async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata() { From 3f0c574de0f24cec224fffc4f63938c6a888e83d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:31:29 +0300 Subject: [PATCH 29/64] test(storage): cover cross-collection migration retry and expiry mismatch Add tests asserting that a cross-collection migration retry succeeds when the target write is byte-identical, including matching schema and expiry, and that a differing expiry is rejected as a conflict while the source record is preserved. The tinybus dev-dependency is repointed at the tinyhumansai fork and the "modules" feature is dropped. Auto-committed-on: dragonfly --- crates/tinyflows-storage/Cargo.toml | 2 +- crates/tinyflows-storage/src/schema_tests.rs | 87 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index 544bb57f..297fae2d 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -22,4 +22,4 @@ data-encoding = "2" [dev-dependencies] tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinybus-sdk = { package = "tinybus", git = "https://github.com/senamakel/tinybus", rev = "c7560289fd6f58b57622234f6d72ff9545cb38c9", default-features = false, features = ["modules"] } +tinybus-sdk = { package = "tinybus", git = "https://github.com/tinyhumansai/tinybus", rev = "d5b949be8ea56eb5a17289a09aa28e8c09bd88b7", default-features = false } diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index 8788c851..df995600 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -103,6 +103,93 @@ async fn schema_migration_uses_metadata_supported_by_the_real_tsd_busproxy() { .is_none()); } +async fn cross_collection_expiry_migration(target_expiry_ms: i64) -> (Arc, Schema) { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + docs.ensure_collection(&CollectionSpec::new("legacy.items")) + .await + .expect("legacy collection"); + docs.ensure_collection(&CollectionSpec::new("flows.archive")) + .await + .expect("archive collection"); + docs.put_with( + "legacy.items", + "same-id", + json!({"value":"before"}), + Precondition::Absent, + WriteOptions::new().schema(1).expires_ms(i64::MAX), + ) + .await + .expect("seed source with absolute expiry"); + docs.put_with( + "flows.archive", + "same-id", + json!({"value":"after"}), + Precondition::Absent, + WriteOptions::new() + .schema(9) + .expires_ms(target_expiry_ms), + ) + .await + .expect("seed target as if first migration write committed"); + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .collection("flows.archive", 9, ScopeClass::Tenant) + .step( + Step::new("archive_v2", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| { + TargetRecord::new("flows.archive", row.id, json!({"value":"after"})) + }) + .collect()) + }) + .source("legacy.items"), + ); + (docs, schema) +} + +#[tokio::test] +async fn cross_collection_retry_accepts_exact_body_schema_and_source_expiry() { + let (docs, schema) = cross_collection_expiry_migration(i64::MAX).await; + schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("a lost acknowledgement can retry the identical target write"); + + let target = docs + .get("flows.archive", "same-id") + .await + .expect("read target") + .expect("target remains"); + assert_eq!(target.doc, json!({"value":"after"})); + assert_eq!(target.meta.schema, 9); + assert_eq!(target.meta.expires_ms, Some(i64::MAX)); + assert!(docs + .get("legacy.items", "same-id") + .await + .expect("read source") + .is_none()); +} + +#[tokio::test] +async fn cross_collection_expiry_mismatch_preserves_the_source_record() { + let (docs, schema) = cross_collection_expiry_migration(i64::MAX - 1).await; + let error = schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect_err("a target with a different expiry is not an identical retry"); + assert_eq!(error.kind(), ErrorKind::Conflict); + let source = docs + .get("legacy.items", "same-id") + .await + .expect("read source") + .expect("source remains until target equivalence is proven"); + assert_eq!(source.meta.expires_ms, Some(i64::MAX)); +} + #[tokio::test] async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata() { let storage = MemoryBackend::new() From 8cf8ea180496da0912f2379585a4740e2a412d9a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:56:46 +0300 Subject: [PATCH 30/64] chore(deps): bump tinystoragedrivers to v2.0.1 Update the tinystoragedrivers crates from v2.0.0 to v2.0.1, which adds the bus and busproxy crates and pulls in a second tinybus revision. The lockfile now tracks both tinybus revisions so the storage driver crates and the rest of the workspace can resolve their respective dependencies. Auto-committed-on: dragonfly --- Cargo.lock | 92 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7363477..25735f20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,6 +2016,27 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "tinybus" +version = "0.1.5" +source = "git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7#d5b949be8ea56eb5a17289a09aa28e8c09bd88b7" +dependencies = [ + "async-trait", + "flate2", + "libc", + "serde", + "serde_json", + "sha2 0.10.9", + "tar", + "tempfile", + "thiserror", + "tokio", + "toml", + "tracing", + "ureq", + "zip", +] + [[package]] name = "tinybus" version = "0.1.5" @@ -2056,7 +2077,7 @@ dependencies = [ "async-trait", "serde", "serde_json", - "tinybus", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", "tokio", "tracing", ] @@ -2178,7 +2199,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "tinybus", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336)", "tinybus-module", "tinyflows", "tinyflows-bus", @@ -2222,7 +2243,9 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.1", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", "tinystoragedrivers-api", + "tinystoragedrivers-busproxy", "tinystoragedrivers-testkit", "tinystoragedrivers-types", "tokio", @@ -2231,8 +2254,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-api" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "bytes", @@ -2245,8 +2268,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-blob" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "bytes", @@ -2257,10 +2280,39 @@ dependencies = [ "tinystoragedrivers-types", ] +[[package]] +name = "tinystoragedrivers-bus" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +dependencies = [ + "serde", + "serde_json", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", + "tinystoragedrivers-types", +] + +[[package]] +name = "tinystoragedrivers-busproxy" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +dependencies = [ + "async-trait", + "bytes", + "futures-core", + "serde", + "serde_json", + "tinybus 0.1.5 (git+https://github.com/tinyhumansai/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7)", + "tinystoragedrivers-api", + "tinystoragedrivers-bus", + "tinystoragedrivers-types", + "tokio", + "zeroize", +] + [[package]] name = "tinystoragedrivers-connect" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "rusqlite", "tinystoragedrivers-types", @@ -2269,8 +2321,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-document" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "data-encoding", @@ -2287,8 +2339,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-kv" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "serde", @@ -2299,8 +2351,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-queue" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "data-encoding", @@ -2315,8 +2367,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-secrets" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "tinystoragedrivers-api", @@ -2326,8 +2378,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-testkit" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "async-trait", "bytes", @@ -2347,8 +2399,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-types" -version = "2.0.0" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.0#e14eef3f52a16b09ed91892686cdadd3b39db56d" +version = "2.0.1" +source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" dependencies = [ "serde", "serde_json", From a7824426b1eb053811f29c4d646a3f2e43bb7c5f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:57:24 +0300 Subject: [PATCH 31/64] fix(storage): preserve source creation time and target expiry in schema writes The target record now carries an absolute expiry instead of a creation timestamp, so writes take the expiry from the target when set and fall back to the source row. The idempotency check compares the existing record's creation time against the source row rather than the target, which keeps replayed writes from being treated as changed. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema.rs | 29 +++++++++++--------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index 0105e40c..f68283c4 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -67,8 +67,8 @@ pub struct TargetRecord { pub id: String, /// Destination body. pub body: Value, - /// Creation timestamp to preserve where available. - pub created_ms: Option, + /// Absolute expiry to preserve or assign to the destination record. + pub expires_ms: Option, } impl TargetRecord { @@ -79,14 +79,14 @@ impl TargetRecord { collection: collection.into(), id: id.into(), body, - created_ms: None, + expires_ms: None, } } - /// Preserve the source creation timestamp. + /// Set the destination's absolute expiry timestamp. #[must_use] - pub fn created_ms(mut self, value: i64) -> Self { - self.created_ms = Some(value); + pub fn expires_ms(mut self, value: i64) -> Self { + self.expires_ms = Some(value); self } } @@ -272,15 +272,12 @@ impl Schema { let precondition = source_row.map_or(Precondition::Absent, |row| { Precondition::Version(row.version) }); - let created_ms = target - .created_ms - .or_else(|| source_row.map(|row| row.meta.created_ms)); let mut options = WriteOptions::new(); options = options.schema(output_schema); - if let Some(created_ms) = created_ms { - options = options.created_ms(created_ms); - } - if let Some(expires_ms) = source_row.and_then(|row| row.meta.expires_ms) { + let expected_expiry = target + .expires_ms + .or_else(|| source_row.and_then(|row| row.meta.expires_ms)); + if let Some(expires_ms) = expected_expiry { options = options.expires_ms(expires_ms); } match docs @@ -301,13 +298,11 @@ impl Schema { ) => { let existing = docs.get(&target.collection, &target.id).await?; - let expected_expiry = - source_row.and_then(|source| source.meta.expires_ms); let identical = existing.is_some_and(|row| { row.doc == target.body && row.meta.schema == output_schema - && created_ms - .is_none_or(|created| row.meta.created_ms == created) + && source_row + .is_none_or(|source| row.meta.created_ms == source.meta.created_ms) && row.meta.expires_ms == expected_expiry }); if !identical { From 7b3b39851e7bcf615acdf1cf02e93db73771538c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:58:10 +0300 Subject: [PATCH 32/64] chore(storage): reformat schema code and tests Reformatted the schema migration check and its tests to satisfy rustfmt, splitting long expressions across lines and reordering imports. The owner-migration test now asserts on expiry metadata instead of a fixed created_ms, matching the metadata the migration actually preserves. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema.rs | 5 +- crates/tinyflows-storage/src/schema_tests.rs | 54 ++++++++++---------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index f68283c4..55d90a52 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -301,8 +301,9 @@ impl Schema { let identical = existing.is_some_and(|row| { row.doc == target.body && row.meta.schema == output_schema - && source_row - .is_none_or(|source| row.meta.created_ms == source.meta.created_ms) + && source_row.is_none_or(|source| { + row.meta.created_ms == source.meta.created_ms + }) && row.meta.expires_ms == expected_expiry }); if !identical { diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index df995600..43c9fabc 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -2,9 +2,9 @@ use super::*; use serde_json::json; use std::sync::Arc; -use tinystoragedrivers_testkit::MemoryBackend; -use tinystoragedrivers_busproxy::{PerCallerEndpoint, StorageServer, connect_storage, serve}; use tinybus_sdk::{Connection, broker::Broker, transport::memory::MemoryBus}; +use tinystoragedrivers_busproxy::{PerCallerEndpoint, StorageServer, connect_storage, serve}; +use tinystoragedrivers_testkit::MemoryBackend; struct BusDocumentsFixture { _broker_task: tokio::task::JoinHandle>, @@ -26,9 +26,8 @@ impl BusDocumentsFixture { .for_scope(&Scope::new("schema-bus").expect("valid test scope")) .expect("memory storage"); let endpoint_storage = backing.clone(); - let endpoint = PerCallerEndpoint::new(move |_| { - Some(StorageServer::new(endpoint_storage.clone())) - }); + let endpoint = + PerCallerEndpoint::new(move |_| Some(StorageServer::new(endpoint_storage.clone()))); let served = serve(&host, Arc::new(endpoint)) .await .expect("serve TSD storage endpoint"); @@ -73,10 +72,7 @@ async fn schema_migration_uses_metadata_supported_by_the_real_tsd_busproxy() { Step::new("items_v2", "flows.items", 1, 2, |rows| { Ok(rows .into_iter() - .map(|row| { - TargetRecord::new("flows.items", row.id, json!({"value":"after"})) - .created_ms(456) - }) + .map(|row| TargetRecord::new("flows.items", row.id, json!({"value":"after"}))) .collect()) }) .source("legacy.items"), @@ -96,14 +92,18 @@ async fn schema_migration_uses_metadata_supported_by_the_real_tsd_busproxy() { assert_eq!(migrated.meta.schema, 2); assert_eq!(migrated.meta.expires_ms, Some(i64::MAX)); assert!(migrated.meta.created_ms > 0); - assert!(backing_docs - .get("legacy.items", "one") - .await - .expect("read old source") - .is_none()); + assert!( + backing_docs + .get("legacy.items", "one") + .await + .expect("read old source") + .is_none() + ); } -async fn cross_collection_expiry_migration(target_expiry_ms: i64) -> (Arc, Schema) { +async fn cross_collection_expiry_migration( + target_expiry_ms: i64, +) -> (Arc, Schema) { let storage = MemoryBackend::new() .for_scope(&Scope::new("local").expect("valid scope")) .expect("memory scope"); @@ -128,9 +128,7 @@ async fn cross_collection_expiry_migration(target_expiry_ms: i64) -> (Arc (Arc 0); + assert_eq!(archived.meta.expires_ms, Some(i64::MAX - 10)); assert_eq!(archived.doc["value"], "archived"); } From e6c3949bf14b75b1fba9cae243a49cfd14337438 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 20 Oct 2026 23:58:48 +0300 Subject: [PATCH 33/64] refactor(storage): separate row lookup from same-row check The row lookup and the check for whether the source and target refer to the same row are now computed separately, so the origin row is available for expiry resolution even when the same-row condition does not hold. This keeps the precondition and expiry logic reading from the same lookup instead of relying on a single combined expression. Auto-committed-on: dragonfly --- crates/tinyflows-storage/src/schema.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/tinyflows-storage/src/schema.rs b/crates/tinyflows-storage/src/schema.rs index 55d90a52..0e906204 100644 --- a/crates/tinyflows-storage/src/schema.rs +++ b/crates/tinyflows-storage/src/schema.rs @@ -266,17 +266,18 @@ impl Schema { )); } let output_schema = self.target_schema(step, &target)?; - let source_row = (source == step.item && target.collection == step.item) - .then(|| page.items.iter().find(|row| row.id == target.id)); - let source_row = source_row.flatten(); - let precondition = source_row.map_or(Precondition::Absent, |row| { + let origin_row = page.items.iter().find(|row| row.id == target.id); + let same_row = (source == step.item && target.collection == step.item) + .then_some(origin_row) + .flatten(); + let precondition = same_row.map_or(Precondition::Absent, |row| { Precondition::Version(row.version) }); let mut options = WriteOptions::new(); options = options.schema(output_schema); let expected_expiry = target .expires_ms - .or_else(|| source_row.and_then(|row| row.meta.expires_ms)); + .or_else(|| origin_row.and_then(|row| row.meta.expires_ms)); if let Some(expires_ms) = expected_expiry { options = options.expires_ms(expires_ms); } @@ -301,7 +302,7 @@ impl Schema { let identical = existing.is_some_and(|row| { row.doc == target.body && row.meta.schema == output_schema - && source_row.is_none_or(|source| { + && same_row.is_none_or(|source| { row.meta.created_ms == source.meta.created_ms }) && row.meta.expires_ms == expected_expiry From 8ff43eb1c5bd51f9b036a3f3d9736c055dd64431 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:01:26 +0300 Subject: [PATCH 34/64] refactor(migrate): drop created_ms metadata from migrated records Migration converters no longer set the record creation timestamp, so the Convert signature and its callers return only the body. Timestamps that were previously promoted to metadata now stay in the body as ordinary fields, and the schedule job converter writes created_ms into the body instead. Auto-committed-on: dragonfly --- .../tinyflows-adaptive/src/drivers/migrate.rs | 16 ++-------- .../tinyflows-drivers/src/catalog/migrate.rs | 31 ++++++++----------- .../tinyflows-drivers/src/schedule/migrate.rs | 12 +++---- crates/tinyflows-drivers/src/store/migrate.rs | 15 ++------- 4 files changed, 21 insertions(+), 53 deletions(-) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index f8c96b16..7060e6ae 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -104,13 +104,7 @@ fn row(source: &SourceRecord) -> Option { return Some(carried(ROWS, source)); }; let body = row_body(rec, text(source, "scope_key"), seq(source)); - let at = body.get("at_ms").and_then(Value::as_i64).unwrap_or(0); - let target = TargetRecord::new(ROWS, source.id.clone(), body); - Some(if at > 0 { - target.created_ms(at) - } else { - target - }) + Some(TargetRecord::new(ROWS, source.id.clone(), body)) } fn lesson(source: &SourceRecord) -> Option { @@ -173,13 +167,7 @@ fn episode(source: &SourceRecord) -> Option { .unwrap_or_default() .to_owned(); let body = episode_body(rec, scope); - let started = body.get("started_ms").and_then(Value::as_i64).unwrap_or(0); - let target = TargetRecord::new(EPISODES, key(&[scope, &id], &source.id), body); - Some(if started > 0 { - target.created_ms(started) - } else { - target - }) + Some(TargetRecord::new(EPISODES, key(&[scope, &id], &source.id), body)) } fn steps(source: &SourceRecord) -> Option { diff --git a/crates/tinyflows-drivers/src/catalog/migrate.rs b/crates/tinyflows-drivers/src/catalog/migrate.rs index 5554f8d4..872af9eb 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate.rs @@ -51,7 +51,7 @@ pub(crate) fn extend_schema(mut schema: Schema) -> Schema { ) } -type Convert = fn(&SourceRecord) -> Option<(Value, Option)>; +type Convert = fn(&SourceRecord) -> Option; /// Converts each record. One that does not convert (a missing JSON field, a /// body that is not an object) is carried unchanged under its v1 id, so it @@ -60,16 +60,11 @@ fn map(batch: Vec, collection: &str, convert: Convert) -> Vec record.created_ms(ms), - None => record, - } + TargetRecord::new(collection, source.id.clone(), body) }) .collect() } @@ -100,7 +95,7 @@ fn carry(body: &Value, out: &mut Map, fields: &[&str]) { } } -fn definition(source: &SourceRecord) -> Option<(Value, Option)> { +fn definition(source: &SourceRecord) -> Option { let body = &source.body; let mut out = Map::new(); carry( @@ -129,10 +124,10 @@ fn definition(source: &SourceRecord) -> Option<(Value, Option)> { if let Some(last) = ms(body, "last_run_at", "last_run_ns") { out.insert("last_run_ms".into(), last.into()); } - Some((Value::Object(out), Some(created))) + Some(Value::Object(out)) } -fn revision(source: &SourceRecord) -> Option<(Value, Option)> { +fn revision(source: &SourceRecord) -> Option { let body = &source.body; let mut out = Map::new(); carry( @@ -152,10 +147,10 @@ fn revision(source: &SourceRecord) -> Option<(Value, Option)> { if let Some(touched) = body.get("touched_ns").and_then(Value::as_i64) { out.insert("touched_ms".into(), (touched / 1_000_000).into()); } - Some((Value::Object(out), Some(created))) + Some(Value::Object(out)) } -fn run(source: &SourceRecord) -> Option<(Value, Option)> { +fn run(source: &SourceRecord) -> Option { let body = &source.body; let mut out = Map::new(); carry( @@ -191,10 +186,10 @@ fn run(source: &SourceRecord) -> Option<(Value, Option)> { if let Some(events) = body.get("events_json") { out.insert("events".into(), parsed(events)); } - Some((Value::Object(out), Some(started))) + Some(Value::Object(out)) } -fn suggestion(source: &SourceRecord) -> Option<(Value, Option)> { +fn suggestion(source: &SourceRecord) -> Option { let body = &source.body; let mut out = Map::new(); carry( @@ -223,10 +218,10 @@ fn suggestion(source: &SourceRecord) -> Option<(Value, Option)> { } let created = ms(body, "created_at", "created_ns").unwrap_or(source.meta.created_ms); out.insert("created_ms".into(), created.into()); - Some((Value::Object(out), Some(created))) + Some(Value::Object(out)) } -fn draft(source: &SourceRecord) -> Option<(Value, Option)> { +fn draft(source: &SourceRecord) -> Option { let whole = parsed(source.body.get("draft_json")?); let whole = whole.as_object()?; let mut out = Map::new(); @@ -255,7 +250,7 @@ fn draft(source: &SourceRecord) -> Option<(Value, Option)> { .unwrap_or(source.meta.created_ms); out.insert("created_ms".into(), created.into()); out.insert("updated_ms".into(), updated.into()); - Some((Value::Object(out), Some(created))) + Some(Value::Object(out)) } #[cfg(test)] diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index b874ab70..9abad6a3 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -2,7 +2,7 @@ //! //! | v1 | v2 | //! |---|---| -//! | `cron_jobs` | `cron.jobs`: `schedule`/`delivery`/`origin` become objects, `next_run`+`next_run_ns` become `next_run_ms`, `created_at`+`created_ms` become the record's creation metadata, `last_run` becomes `last_run_ms` | +//! | `cron_jobs` | `cron.jobs`: `schedule`/`delivery`/`origin` become objects, `next_run`+`next_run_ns` become `next_run_ms`, `created_at`+`created_ms` become the body field `created_ms`, `last_run` becomes `last_run_ms` | //! | `cron_runs` | `cron.runs`: `started_at`+`started_ns` become `started_ms`, `finished_at` becomes `finished_ms` | //! | `cron_counters` `admission` | `cron.admission` `ledger` (members as a list) | //! | `cron_counters` `runs` | `tsd.counters` owner `cron` name `runs`, through [`import_legacy_cron_counters`] | @@ -165,12 +165,8 @@ fn job(source: &SourceRecord) -> Option { .and_then(instant_ms) .or_else(|| body.get("created_ms").and_then(Value::as_i64)) .unwrap_or(source.meta.created_ms); - let record = TargetRecord::new(JOBS, source.id.clone(), Value::Object(out)); - Some(if created > 0 { - record.created_ms(created) - } else { - record - }) + out.insert("created_ms".into(), created.into()); + Some(TargetRecord::new(JOBS, source.id.clone(), Value::Object(out))) } fn run(source: &SourceRecord) -> Option { @@ -201,7 +197,7 @@ fn run(source: &SourceRecord) -> Option { "finished_ms".into(), ms("finished_at", "finished_ns")?.into(), ); - Some(TargetRecord::new(RUNS, source.id.clone(), Value::Object(out)).created_ms(started)) + Some(TargetRecord::new(RUNS, source.id.clone(), Value::Object(out))) } /// `cron_counters` holds two documents: the run counter and the admission diff --git a/crates/tinyflows-drivers/src/store/migrate.rs b/crates/tinyflows-drivers/src/store/migrate.rs index 5b7c5c79..64991d27 100644 --- a/crates/tinyflows-drivers/src/store/migrate.rs +++ b/crates/tinyflows-drivers/src/store/migrate.rs @@ -129,12 +129,7 @@ fn revision(source: &SourceRecord) -> Option { }; body.insert("workflow_id".into(), json!(workflow_id)); body.insert("superseded_ms".into(), json!(superseded)); - let target = TargetRecord::new(REVISIONS, source.id.clone(), Value::Object(body)); - Some(if superseded > 0 { - target.created_ms(superseded) - } else { - target - }) + Some(TargetRecord::new(REVISIONS, source.id.clone(), Value::Object(body))) } fn run(source: &SourceRecord) -> Option { @@ -158,13 +153,7 @@ fn run(source: &SourceRecord) -> Option { return None; }; body.insert("settled".into(), json!(settled)); - let started = body.get("started_ms").and_then(Value::as_i64).unwrap_or(0); - let target = TargetRecord::new(RUNS, key(&id, &source.id), Value::Object(body)); - Some(if started > 0 { - target.created_ms(started) - } else { - target - }) + Some(TargetRecord::new(RUNS, key(&id, &source.id), Value::Object(body))) } fn journal(source: &SourceRecord) -> Option { From 850f21fca593d854cfce18deba1ade62fa9a8c70 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:01:47 +0300 Subject: [PATCH 35/64] fix(schedule): store imported job creation time in the document body Imported cron jobs now carry their original creation time in the document body instead of relying on backend record metadata, since module document ports cannot set host-owned metadata. Reading a job prefers the body value and falls back to the backend metadata for ordinary jobs. Auto-committed-on: dragonfly --- .../src/schedule/admission.rs | 19 ++++++------------- .../tinyflows-drivers/src/schedule/codec.rs | 9 ++++++--- .../src/schedule/migrate_tests.rs | 7 ++----- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/crates/tinyflows-drivers/src/schedule/admission.rs b/crates/tinyflows-drivers/src/schedule/admission.rs index 435eebba..23081708 100644 --- a/crates/tinyflows-drivers/src/schedule/admission.rs +++ b/crates/tinyflows-drivers/src/schedule/admission.rs @@ -189,13 +189,14 @@ impl CronDocuments { pub(super) async fn insert_with_created( &self, id: &str, - doc: Value, + mut doc: Value, created_ms: i64, ) -> Result<()> { - let mut options = tinyflows_storage::WriteOptions::new().schema(super::SCHEMA_VERSION); - if created_ms > 0 { - options = options.created_ms(created_ms); - } + let body = doc + .as_object_mut() + .ok_or_else(|| anyhow!("cron store: imported job must be an object"))?; + body.insert("created_ms".into(), created_ms.into()); + let options = tinyflows_storage::WriteOptions::new().schema(super::SCHEMA_VERSION); let member = member(id, &doc)?; // A nontransactional backend cannot enrol, so check the persisted // policy before the job exists rather than after it is written. @@ -209,14 +210,6 @@ impl CronDocuments { .await { Ok(_) => {} - // A store that cannot set a creation time (a module's document - // callback) files the job under the time of the import instead. - Err(error) if matches!(error.kind(), ErrorKind::Unsupported(_)) => { - return self - .commit_job(id, None, Some(doc), vec![]) - .await - .map(|_| ()); - } Err(error) => return Err(storage_error(error)), } if !transactional { diff --git a/crates/tinyflows-drivers/src/schedule/codec.rs b/crates/tinyflows-drivers/src/schedule/codec.rs index 1a576b1a..9894d4db 100644 --- a/crates/tinyflows-drivers/src/schedule/codec.rs +++ b/crates/tinyflows-drivers/src/schedule/codec.rs @@ -164,8 +164,9 @@ pub(super) fn set_last_run( doc.insert("last_output".into(), json!(output)); } -/// The job a stored document holds. Its creation time is the document's -/// `created_ms` metadata. +/// The job a stored document holds. Imported jobs retain their original +/// creation time in the body because module document ports cannot set the +/// host-owned record metadata; ordinary jobs still use backend metadata. pub(super) fn doc_to_job(stored: &Versioned) -> Result { let doc = &stored.doc; let schedule = schedule_from_value( @@ -215,7 +216,9 @@ pub(super) fn doc_to_job(stored: &Versioned) -> Result { enabled: flag("enabled")?, delivery, delete_after_run: flag("delete_after_run")?, - created_at: from_ms(stored.meta.created_ms)?, + created_at: from_ms( + integer(doc, "created_ms")?.unwrap_or(stored.meta.created_ms), + )?, next_run: from_ms( integer(doc, "next_run_ms")? .ok_or_else(|| anyhow!("cron store: document has no `next_run_ms`"))?, diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index 878ef8ba..78c49dc2 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -120,13 +120,10 @@ async fn v1_jobs_runs_and_the_ledger_migrate_and_read_back() { assert_eq!(j1.doc["origin"], json!({"kind": "web", "thread_id": "t"})); assert_eq!(j1.doc["next_run_ms"], 1_772_701_200_000_i64); assert_eq!(j1.doc["last_run_ms"], 1_772_614_800_000_i64); - assert_eq!( - j1.meta.created_ms, 1_772_600_767_123_i64, - "creation time is metadata" - ); + assert_eq!(j1.doc["created_ms"], 1_772_600_767_123_i64); + assert!(j1.meta.created_ms > 0, "backend owns record metadata"); for gone in [ "created_at", - "created_ms", "next_run", "next_run_ns", "last_run", From 074078fb222ff49388ebe8c0fb7a2714f8755945 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:02:02 +0300 Subject: [PATCH 36/64] test: assert backend-owned record metadata instead of source times Migration tests no longer expect the record metadata creation time to match the source document's timestamp, since the backend now assigns it. The admission doc comment was updated to reflect that the original creation time lives in the document body rather than the metadata. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/catalog/migrate_tests.rs | 8 ++++---- crates/tinyflows-drivers/src/schedule/admission.rs | 4 ++-- crates/tinyflows-drivers/src/store/migrate_tests.rs | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index a3fe86b3..aa8b6a41 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -325,7 +325,7 @@ async fn catalog_migration_preserves_source_record_metadata_without_body_timesta .unwrap() .unwrap(); assert_eq!(revision.doc["created_ms"], created_ms); - assert_eq!(revision.meta.created_ms, created_ms); + assert!(revision.meta.created_ms > 0, "backend owns record metadata"); } #[tokio::test] @@ -370,7 +370,7 @@ async fn catalog_migration_preserves_zero_negative_and_source_creation_times() { .unwrap() .unwrap(); assert_eq!(epoch.doc["created_ms"], 0); - assert_eq!(epoch.meta.created_ms, 0); + assert!(epoch.meta.created_ms > 0, "backend owns record metadata"); let negative = storage .documents() .get("flows.revisions", "negative-revision") @@ -378,7 +378,7 @@ async fn catalog_migration_preserves_zero_negative_and_source_creation_times() { .unwrap() .unwrap(); assert_eq!(negative.doc["created_ms"], -1); - assert_eq!(negative.meta.created_ms, -1); + assert!(negative.meta.created_ms > 0, "backend owns record metadata"); let draft = storage .documents() .get("flows.drafts", "older-draft") @@ -387,5 +387,5 @@ async fn catalog_migration_preserves_zero_negative_and_source_creation_times() { .unwrap(); assert_eq!(draft.doc["created_ms"], draft_created_ms); assert_eq!(draft.doc["updated_ms"], draft_updated_ms); - assert_eq!(draft.meta.created_ms, draft_created_ms); + assert!(draft.meta.created_ms > 0, "backend owns record metadata"); } diff --git a/crates/tinyflows-drivers/src/schedule/admission.rs b/crates/tinyflows-drivers/src/schedule/admission.rs index 23081708..7b4853be 100644 --- a/crates/tinyflows-drivers/src/schedule/admission.rs +++ b/crates/tinyflows-drivers/src/schedule/admission.rs @@ -180,8 +180,8 @@ impl CronDocuments { bail!("cron store: admission initialization kept changing") } - /// Writes a new job with an explicit creation time (an importer keeps the - /// original) and enrols it in the admission ledger. The job is written + /// Writes a new job with its original creation time in the document body + /// and enrols it in the admission ledger. The job is written /// first and enrolled second, so a job over the cap is taken back. /// /// # Errors diff --git a/crates/tinyflows-drivers/src/store/migrate_tests.rs b/crates/tinyflows-drivers/src/store/migrate_tests.rs index c3eac5b9..e6e0aab6 100644 --- a/crates/tinyflows-drivers/src/store/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/store/migrate_tests.rs @@ -162,7 +162,7 @@ async fn a_v1_revision_keeps_its_id_and_time() { assert_eq!(stored.doc["superseded_ms"], 7); assert_eq!(stored.doc["description"], "old"); assert!(stored.doc.get("id").is_none() && stored.doc.get("superseded_at").is_none()); - assert_eq!(stored.meta.created_ms, 7, "history keeps its time"); + assert!(stored.meta.created_ms > 0, "backend owns record metadata"); let store = DriverWorkflowStore::new(docs(&storage)); let history = tokio::task::spawn_blocking(move || store.list_revisions("sweep")) .await @@ -213,7 +213,7 @@ async fn a_v1_run_becomes_snake_case_with_ms_times() { "free-form output is verbatim" ); assert_eq!(stored.doc["settled"], true); - assert_eq!(stored.meta.created_ms, 1_700_000_000_123_i64); + assert!(stored.meta.created_ms > 0, "backend owns record metadata"); let store = DriverWorkflowStore::new(docs(&storage)); let read = tokio::task::spawn_blocking(move || store.get_run("run-1")) .await From 71213799d1cafcf275138eb26ae20c611280cff2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:02:21 +0300 Subject: [PATCH 37/64] docs(schedule): document created_ms body override The schedule codec and module docs now note that a job's creation time normally comes from document metadata, but imported or migrated jobs keep their original time in a body created_ms field because module ports cannot set host-owned metadata. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/schedule/codec.rs | 5 +++-- crates/tinyflows-drivers/src/schedule/mod.rs | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-drivers/src/schedule/codec.rs b/crates/tinyflows-drivers/src/schedule/codec.rs index 9894d4db..8e5d1d7c 100644 --- a/crates/tinyflows-drivers/src/schedule/codec.rs +++ b/crates/tinyflows-drivers/src/schedule/codec.rs @@ -3,8 +3,9 @@ //! Bodies are bare snake_case objects. Nested values (`schedule`, `delivery`, //! `origin`, `policy`) are real objects, not JSON strings; every instant is one //! `_ms` field (`next_run_ms`, `last_run_ms`, `started_ms`, `finished_ms`), and -//! a job's creation time is the document's `created_ms` metadata, not a body -//! field. Optional fields are left out rather than stored as `null`. A job +//! a job's creation time normally comes from document metadata, with a body +//! `created_ms` override for imported or migrated jobs. Optional fields are +//! left out rather than stored as `null`. A job //! document that cannot be read back is an error naming the field, never a //! silently defaulted job. diff --git a/crates/tinyflows-drivers/src/schedule/mod.rs b/crates/tinyflows-drivers/src/schedule/mod.rs index 09bbe190..c7033be4 100644 --- a/crates/tinyflows-drivers/src/schedule/mod.rs +++ b/crates/tinyflows-drivers/src/schedule/mod.rs @@ -16,7 +16,9 @@ //! //! Run numbers come from `tsd.counters` (owner `cron`, name `runs`), occurrence //! claims are `tsd.leases` (owner `cron`, key `Key::of([job_id, occurrence_ms])`) -//! and a job's creation time is the document's `created_ms` metadata. A job's +//! and a job's creation time is normally the document's `created_ms` metadata. +//! Imported and migrated jobs retain their original time in the body field +//! `created_ms`, since module ports cannot set host-owned metadata. A job's //! `schedule`, `delivery`, `origin` and `policy` are real objects. Optional //! fields are left out rather than stored as `null`. Every other instant is one //! epoch-millisecond field: `next_run_ms`, `last_run_ms`, `started_ms`, From 62306904220b67e9fdac75a06e61121d62080b26 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:02:52 +0300 Subject: [PATCH 38/64] fix(tests): reject module-supplied created_ms in storage callbacks The document and catalog test callbacks no longer forward a created_ms value from module requests into write options, returning an invalid input error instead. This keeps created_ms a host-only migration option so modules cannot backdate documents through the callback surface. Auto-committed-on: dragonfly --- .../tinyflows-module/tests/support/document_callbacks.rs | 8 +++++--- .../tests/support/storage_migration_artifact_tests.rs | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/tinyflows-module/tests/support/document_callbacks.rs b/crates/tinyflows-module/tests/support/document_callbacks.rs index 89f4cd97..e6f266db 100644 --- a/crates/tinyflows-module/tests/support/document_callbacks.rs +++ b/crates/tinyflows-module/tests/support/document_callbacks.rs @@ -53,6 +53,11 @@ impl Documents { } "get" => encode(self.docs.get(collection, id).await?), "put" => { + if request["created_ms"].as_i64().is_some() { + return Err(StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + )); + } let mut options = tinyflows_storage::WriteOptions::new(); if let Some(schema) = request["schema"].as_u64() { options = options.schema(u32::try_from(schema).unwrap_or(0)); @@ -60,9 +65,6 @@ impl Documents { if let Some(expires_ms) = request["expires_ms"].as_i64() { options = options.expires_ms(expires_ms); } - if let Some(created_ms) = request["created_ms"].as_i64() { - options = options.created_ms(created_ms); - } encode( self.docs .put_with( diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 765ad07e..7135a384 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -48,10 +48,12 @@ impl CatalogCallback { } async fn doc_put(&self, request: Value) -> tinybus::Result { - let mut options = WriteOptions::new(); - if let Some(created_ms) = request["created_ms"].as_i64() { - options = options.created_ms(created_ms); + if request["created_ms"].as_i64().is_some() { + return Ok(catalog_reply(Err(tinyflows_storage::StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + )))); } + let mut options = WriteOptions::new(); if let Some(schema) = request["schema"].as_u64() { options = options.schema(u32::try_from(schema).unwrap_or(0)); } From 4afdfd342c91af92dff4c5972360f90577fda1fb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:03:05 +0300 Subject: [PATCH 39/64] chore: apply rustfmt formatting to migration code Reformat long expressions and function calls across the migration drivers and their test support so they match rustfmt output. No behaviour changes. Auto-committed-on: dragonfly --- crates/tinyflows-adaptive/src/drivers/migrate.rs | 6 +++++- crates/tinyflows-drivers/src/schedule/codec.rs | 4 +--- crates/tinyflows-drivers/src/schedule/migrate.rs | 12 ++++++++++-- crates/tinyflows-drivers/src/store/migrate.rs | 12 ++++++++++-- .../support/storage_migration_artifact_tests.rs | 8 +++++--- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index 7060e6ae..48c3e508 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -167,7 +167,11 @@ fn episode(source: &SourceRecord) -> Option { .unwrap_or_default() .to_owned(); let body = episode_body(rec, scope); - Some(TargetRecord::new(EPISODES, key(&[scope, &id], &source.id), body)) + Some(TargetRecord::new( + EPISODES, + key(&[scope, &id], &source.id), + body, + )) } fn steps(source: &SourceRecord) -> Option { diff --git a/crates/tinyflows-drivers/src/schedule/codec.rs b/crates/tinyflows-drivers/src/schedule/codec.rs index 8e5d1d7c..bf8922fc 100644 --- a/crates/tinyflows-drivers/src/schedule/codec.rs +++ b/crates/tinyflows-drivers/src/schedule/codec.rs @@ -217,9 +217,7 @@ pub(super) fn doc_to_job(stored: &Versioned) -> Result { enabled: flag("enabled")?, delivery, delete_after_run: flag("delete_after_run")?, - created_at: from_ms( - integer(doc, "created_ms")?.unwrap_or(stored.meta.created_ms), - )?, + created_at: from_ms(integer(doc, "created_ms")?.unwrap_or(stored.meta.created_ms))?, next_run: from_ms( integer(doc, "next_run_ms")? .ok_or_else(|| anyhow!("cron store: document has no `next_run_ms`"))?, diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index 9abad6a3..efc5f889 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -166,7 +166,11 @@ fn job(source: &SourceRecord) -> Option { .or_else(|| body.get("created_ms").and_then(Value::as_i64)) .unwrap_or(source.meta.created_ms); out.insert("created_ms".into(), created.into()); - Some(TargetRecord::new(JOBS, source.id.clone(), Value::Object(out))) + Some(TargetRecord::new( + JOBS, + source.id.clone(), + Value::Object(out), + )) } fn run(source: &SourceRecord) -> Option { @@ -197,7 +201,11 @@ fn run(source: &SourceRecord) -> Option { "finished_ms".into(), ms("finished_at", "finished_ns")?.into(), ); - Some(TargetRecord::new(RUNS, source.id.clone(), Value::Object(out))) + Some(TargetRecord::new( + RUNS, + source.id.clone(), + Value::Object(out), + )) } /// `cron_counters` holds two documents: the run counter and the admission diff --git a/crates/tinyflows-drivers/src/store/migrate.rs b/crates/tinyflows-drivers/src/store/migrate.rs index 64991d27..e45face0 100644 --- a/crates/tinyflows-drivers/src/store/migrate.rs +++ b/crates/tinyflows-drivers/src/store/migrate.rs @@ -129,7 +129,11 @@ fn revision(source: &SourceRecord) -> Option { }; body.insert("workflow_id".into(), json!(workflow_id)); body.insert("superseded_ms".into(), json!(superseded)); - Some(TargetRecord::new(REVISIONS, source.id.clone(), Value::Object(body))) + Some(TargetRecord::new( + REVISIONS, + source.id.clone(), + Value::Object(body), + )) } fn run(source: &SourceRecord) -> Option { @@ -153,7 +157,11 @@ fn run(source: &SourceRecord) -> Option { return None; }; body.insert("settled".into(), json!(settled)); - Some(TargetRecord::new(RUNS, key(&id, &source.id), Value::Object(body))) + Some(TargetRecord::new( + RUNS, + key(&id, &source.id), + Value::Object(body), + )) } fn journal(source: &SourceRecord) -> Option { diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 7135a384..0d1ab9a6 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -49,9 +49,11 @@ impl CatalogCallback { async fn doc_put(&self, request: Value) -> tinybus::Result { if request["created_ms"].as_i64().is_some() { - return Ok(catalog_reply(Err(tinyflows_storage::StorageError::invalid_input( - "a module cannot set created_ms: it is a host-only migration option", - )))); + return Ok(catalog_reply(Err( + tinyflows_storage::StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + ), + ))); } let mut options = WriteOptions::new(); if let Some(schema) = request["schema"].as_u64() { From 2efc25bcf14986c52a080be138bb34bce090c83e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:03:16 +0300 Subject: [PATCH 40/64] test(catalog): reject created_ms in module doc_put The test host now returns an invalid input error when a module supplies created_ms, since that option is reserved for host-side migrations, and the corresponding option plumbing was dropped. Auto-committed-on: dragonfly --- crates/tinyflows-module/src/catalog_service_tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index bdd02937..8e9d840e 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -44,6 +44,11 @@ impl HostDocuments { } async fn doc_put(&self, request: Value) -> tinybus::Result { + if request["created_ms"].as_i64().is_some() { + return Ok(reply(Err(tinyflows_storage::StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + )))); + } if request["collection"] == "flows.revisions" && self.fail_revision_put.swap(false, Ordering::SeqCst) { @@ -55,9 +60,6 @@ impl HostDocuments { if let Some(schema) = request["schema"].as_u64() { options = options.schema(u32::try_from(schema).unwrap_or(0)); } - if let Some(created_ms) = request["created_ms"].as_i64() { - options = options.created_ms(created_ms); - } let result = self .documents .put_with( From 8af0ad40c1198eb9f18db0db7644465309e992e4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:03:33 +0300 Subject: [PATCH 41/64] fix(module): reject created_ms in module document writes Modules can no longer set created_ms on put, since it is a host-only migration option; attempts now fail with an invalid input error. The catalog path also stops forwarding the field, and the schedule path no longer serializes it into the request. Auto-committed-on: dragonfly --- crates/tinyflows-module/src/catalog_documents.rs | 7 ++++++- crates/tinyflows-module/src/schedule_documents.rs | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/tinyflows-module/src/catalog_documents.rs b/crates/tinyflows-module/src/catalog_documents.rs index 0a58b12b..c2069e4a 100644 --- a/crates/tinyflows-module/src/catalog_documents.rs +++ b/crates/tinyflows-module/src/catalog_documents.rs @@ -270,6 +270,11 @@ impl DocumentStore for Documents { precondition: Precondition, options: WriteOptions, ) -> Result { + if options.created_ms.is_some() { + return Err(StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + )); + } self.call_record( storage::PUT, collection, @@ -281,7 +286,7 @@ impl DocumentStore for Documents { precondition: serde_json::from_value(serde_json::to_value(precondition)?)?, schema: options.schema, expires_ms: options.expires_ms, - created_ms: options.created_ms, + created_ms: None, }, ) .await diff --git a/crates/tinyflows-module/src/schedule_documents.rs b/crates/tinyflows-module/src/schedule_documents.rs index 8442bc5d..5033cd22 100644 --- a/crates/tinyflows-module/src/schedule_documents.rs +++ b/crates/tinyflows-module/src/schedule_documents.rs @@ -232,6 +232,11 @@ impl DocumentStore for Documents { precondition: Precondition, options: WriteOptions, ) -> Result { + if options.created_ms.is_some() { + return Err(StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + )); + } let mut request = json!({"op":"put","collection":collection,"id":id,"doc":doc,"precondition":precondition}); if options.schema != 0 { request["schema"] = json!(options.schema); @@ -239,9 +244,6 @@ impl DocumentStore for Documents { if let Some(expires_ms) = options.expires_ms { request["expires_ms"] = json!(expires_ms); } - if let Some(created_ms) = options.created_ms { - request["created_ms"] = json!(created_ms); - } self.call_record(collection, id, request).await } async fn delete(&self, collection: &str, id: &str, precondition: Precondition) -> Result { From 5d92e9847687f06b5dfe7e17bd2fa75712c7ba26 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:04:00 +0300 Subject: [PATCH 42/64] test(catalog): cover rejection of host-only creation metadata Add a test asserting that the document bridge refuses writes carrying host-only creation metadata, so module ports cannot forward it to the host. Auto-committed-on: dragonfly --- .../src/catalog_documents_tests.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/crates/tinyflows-module/src/catalog_documents_tests.rs b/crates/tinyflows-module/src/catalog_documents_tests.rs index 71e51fc6..8a69f320 100644 --- a/crates/tinyflows-module/src/catalog_documents_tests.rs +++ b/crates/tinyflows-module/src/catalog_documents_tests.rs @@ -17,3 +17,33 @@ fn shared_catalog_records_are_key_fenced_and_never_scanned() { assert!(Documents::validate_collection("flows_revisions").is_ok()); assert!(Documents::validate_collection("unrelated").is_err()); } + +#[tokio::test] +async fn catalog_document_bridge_rejects_host_only_creation_metadata() { + let bus = tinybus::transport::memory::MemoryBus::new(); + tinybus::broker::Broker::new().spawn(bus.clone()); + let connection = tinybus::Connection::connect(bus.connect().await.unwrap()) + .await + .unwrap(); + let documents = Documents::new( + connection, + CallbackAddress { + destination: "missing.Host".into(), + object_path: "/missing/Host".into(), + }, + ) + .await + .unwrap(); + + let error = documents + .put_with( + "flows.revisions", + "revision", + serde_json::json!({"created_ms": 1}), + Precondition::Absent, + WriteOptions::new().created_ms(1), + ) + .await + .expect_err("module ports must not forward host-only creation metadata"); + assert_eq!(error.kind(), tinyflows_storage::ErrorKind::InvalidInput); +} From 7bb8e55fd4146ce7071ae42eff64f4af8b6365c1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:04:14 +0300 Subject: [PATCH 43/64] test(schedule_documents): assert forged creation metadata is rejected before callback The document bridge test now counts callback invocations and verifies that a put carrying host-owned created_ms metadata fails with InvalidInput without reaching the callback, confirming the module rejects the request before delegating to the host. Auto-committed-on: dragonfly --- .../src/schedule_documents_tests.rs | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index 0364358f..f21d7cac 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -4,16 +4,21 @@ use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows_storage::{Capability, ErrorKind, Scope, StorageBackend, WriteOp}; -struct Callback(Arc); +struct Callback { + docs: Arc, + calls: Arc, +} #[tinybus::interface(name = "ai.tinyhumans.tinyflows.ScheduleDocuments")] impl Callback { async fn document(&self, request: Value) -> tinybus::Result { + self.calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let value = match request["op"].as_str().unwrap() { "capabilities" => Ok( - json!({"transactions": self.0.capabilities().contains(Capability::Transactions)}), + json!({"transactions": self.docs.capabilities().contains(Capability::Transactions)}), ), "atomic_batch" => self - .0 + .docs .atomic_batch(serde_json::from_value(request["ops"].clone()).unwrap()) .await .map(|results| serde_json::to_value(results).unwrap()), @@ -50,7 +55,10 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( connection .serve_at( "/test/Documents".try_into().unwrap(), - Callback(storage.clone()), + Callback { + docs: storage.clone(), + calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }, ) .await .unwrap(); @@ -65,6 +73,23 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( .await .unwrap(); assert!(documents.capabilities().contains(Capability::Transactions)); + let calls_before_host_metadata_request = callbacks.calls.load(std::sync::atomic::Ordering::SeqCst); + let metadata_error = documents + .put_with( + "cron.jobs", + "job-with-forged-created-time", + json!({"enabled":true}), + Precondition::Absent, + tinyflows_storage::WriteOptions::new().created_ms(1), + ) + .await + .expect_err("module must refuse host-owned creation metadata"); + assert_eq!(metadata_error.kind(), ErrorKind::InvalidInput); + assert_eq!( + callbacks.calls.load(std::sync::atomic::Ordering::SeqCst), + calls_before_host_metadata_request, + "the forbidden metadata must be rejected before the callback" + ); let put = |collection: &str, precondition| WriteOp::Put { collection: collection.into(), id: "job".into(), From a4239f671a4708da9d5aba4001ed0e31b2abe636 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:04:53 +0300 Subject: [PATCH 44/64] test(schedule-documents): hoist callback into a binding Extract the callback into a named binding so the test can inspect its call count, and derive Clone on the callback struct to allow it to be served and retained. The remaining changes are formatting only. Auto-committed-on: dragonfly --- .../src/schedule_documents_tests.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/tinyflows-module/src/schedule_documents_tests.rs b/crates/tinyflows-module/src/schedule_documents_tests.rs index f21d7cac..7bb32e2c 100644 --- a/crates/tinyflows-module/src/schedule_documents_tests.rs +++ b/crates/tinyflows-module/src/schedule_documents_tests.rs @@ -4,6 +4,7 @@ use tinystoragedrivers_testkit::MemoryBackend as MemoryStorage; use std::sync::Arc; use tinyflows_storage::{Capability, ErrorKind, Scope, StorageBackend, WriteOp}; +#[derive(Clone)] struct Callback { docs: Arc, calls: Arc, @@ -11,8 +12,7 @@ struct Callback { #[tinybus::interface(name = "ai.tinyhumans.tinyflows.ScheduleDocuments")] impl Callback { async fn document(&self, request: Value) -> tinybus::Result { - self.calls - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); let value = match request["op"].as_str().unwrap() { "capabilities" => Ok( json!({"transactions": self.docs.capabilities().contains(Capability::Transactions)}), @@ -52,14 +52,12 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( .ensure_collection(&CollectionSpec::new("other_domain")) .await .unwrap(); + let callbacks = Callback { + docs: storage.clone(), + calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }; connection - .serve_at( - "/test/Documents".try_into().unwrap(), - Callback { - docs: storage.clone(), - calls: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - }, - ) + .serve_at("/test/Documents".try_into().unwrap(), callbacks.clone()) .await .unwrap(); connection.request_name("test.Documents").await.unwrap(); @@ -73,7 +71,8 @@ async fn document_bridge_preserves_atomic_writes_conflicts_and_collection_scope( .await .unwrap(); assert!(documents.capabilities().contains(Capability::Transactions)); - let calls_before_host_metadata_request = callbacks.calls.load(std::sync::atomic::Ordering::SeqCst); + let calls_before_host_metadata_request = + callbacks.calls.load(std::sync::atomic::Ordering::SeqCst); let metadata_error = documents .put_with( "cron.jobs", From 3081fa03f7451467b97f046e7ad335a911fd3755 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:05:24 +0300 Subject: [PATCH 45/64] test: annotate error type in doc_put test helpers Add explicit `Err::<(), _>` turbofish annotations to the error construction in two test helpers so the reply type is inferred without ambiguity. Auto-committed-on: dragonfly --- crates/tinyflows-module/src/catalog_service_tests.rs | 2 +- .../tests/support/storage_migration_artifact_tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index 8e9d840e..0801fcb8 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -45,7 +45,7 @@ impl HostDocuments { async fn doc_put(&self, request: Value) -> tinybus::Result { if request["created_ms"].as_i64().is_some() { - return Ok(reply(Err(tinyflows_storage::StorageError::invalid_input( + return Ok(reply(Err::<(), _>(tinyflows_storage::StorageError::invalid_input( "a module cannot set created_ms: it is a host-only migration option", )))); } diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 0d1ab9a6..66286b9c 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -49,7 +49,7 @@ impl CatalogCallback { async fn doc_put(&self, request: Value) -> tinybus::Result { if request["created_ms"].as_i64().is_some() { - return Ok(catalog_reply(Err( + return Ok(catalog_reply(Err::<(), _>( tinyflows_storage::StorageError::invalid_input( "a module cannot set created_ms: it is a host-only migration option", ), From 65244ae468142611468f521ad38a617f9493d56a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:06:07 +0300 Subject: [PATCH 46/64] test(catalog): reformat created_ms rejection in host documents stub Reindented the error construction in the test host's doc_put stub so the nested call reads cleanly. No behaviour change. Auto-committed-on: dragonfly --- crates/tinyflows-module/src/catalog_service_tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-module/src/catalog_service_tests.rs b/crates/tinyflows-module/src/catalog_service_tests.rs index 0801fcb8..741bd7ea 100644 --- a/crates/tinyflows-module/src/catalog_service_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_tests.rs @@ -45,9 +45,11 @@ impl HostDocuments { async fn doc_put(&self, request: Value) -> tinybus::Result { if request["created_ms"].as_i64().is_some() { - return Ok(reply(Err::<(), _>(tinyflows_storage::StorageError::invalid_input( - "a module cannot set created_ms: it is a host-only migration option", - )))); + return Ok(reply(Err::<(), _>( + tinyflows_storage::StorageError::invalid_input( + "a module cannot set created_ms: it is a host-only migration option", + ), + ))); } if request["collection"] == "flows.revisions" && self.fail_revision_put.swap(false, Ordering::SeqCst) From 7447a09413d25ed229c335414e400444ce9e9a3b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:06:30 +0300 Subject: [PATCH 47/64] fix(storage): preserve source expiry when migrating records Migration targets now carry the source record's absolute expiry, so renamed or re-collectioned records keep their original TTL instead of losing it. A helper on TargetRecord copies the expiry from the source, and the catalog migration applies it to both converted and unconvertible records. Auto-committed-on: dragonfly --- .../tinyflows-drivers/src/catalog/migrate.rs | 5 +- crates/tinyflows-storage/src/schema.rs | 8 +++ crates/tinyflows-storage/src/schema_tests.rs | 52 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/crates/tinyflows-drivers/src/catalog/migrate.rs b/crates/tinyflows-drivers/src/catalog/migrate.rs index 872af9eb..b2b202c5 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate.rs @@ -62,9 +62,10 @@ fn map(batch: Vec, collection: &str, convert: Convert) -> Vec Self { + self.expires_ms = source.meta.expires_ms; + self + } } type StepFn = dyn Fn(Vec) -> Result> + Send + Sync; diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index 43c9fabc..f2d05bb5 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -187,6 +187,58 @@ async fn cross_collection_expiry_mismatch_preserves_the_source_record() { assert_eq!(source.meta.expires_ms, Some(i64::MAX)); } +#[tokio::test] +async fn renamed_migration_targets_keep_their_source_absolute_expiry() { + let storage = MemoryBackend::new() + .for_scope(&Scope::new("local").expect("valid scope")) + .expect("memory scope"); + let docs = Arc::clone(storage.documents()); + docs.ensure_collection(&CollectionSpec::new("legacy.items")) + .await + .expect("legacy source collection"); + let expiry = i64::MAX - 100; + docs.put_with( + "legacy.items", + "old-id", + json!({"value":"before"}), + Precondition::Absent, + WriteOptions::new().schema(1).expires_ms(expiry), + ) + .await + .expect("seed source with finite absolute expiry"); + + let schema = Schema::new(Owner::new("flows").expect("valid owner")) + .collection("flows.items", 2, ScopeClass::Tenant) + .step( + Step::new("rename_items_v2", "flows.items", 1, 2, |rows| { + Ok(rows + .into_iter() + .map(|row| { + TargetRecord::new("flows.items", "new-id", json!({"value":"after"})) + .preserve_expiry_from(&row) + }) + .collect()) + }) + .source("legacy.items"), + ); + schema + .migrate(&docs, &MigrateOptions::default()) + .await + .expect("renamed target migration succeeds"); + + let target = docs + .get("flows.items", "new-id") + .await + .expect("read renamed target") + .expect("renamed target exists"); + assert_eq!(target.meta.expires_ms, Some(expiry)); + assert!(docs + .get("legacy.items", "old-id") + .await + .expect("read source") + .is_none()); +} + #[tokio::test] async fn owner_migration_rewrites_in_place_once_and_preserves_document_metadata() { let storage = MemoryBackend::new() From 374d8b121eb8fd3b533c86b0f3fa2ff2d650550e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:06:39 +0300 Subject: [PATCH 48/64] fix(schedule): preserve record expiry during v1 migration Migrated records now carry over their original expiry instead of dropping it, so TTLs set on v1 documents survive the upgrade. This applies to jobs, runs, admission counters, and unconvertible records that are carried forward as-is. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/schedule/migrate.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index efc5f889..ca8ebdad 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -72,6 +72,7 @@ fn map( convert(source).unwrap_or_else(|| { tracing::warn!(collection, id = %source.id, "[cron-storage] carrying unconvertible v1 record"); TargetRecord::new(collection, source.id.clone(), source.body.clone()) + .preserve_expiry_from(source) }) }) .collect() @@ -170,7 +171,8 @@ fn job(source: &SourceRecord) -> Option { JOBS, source.id.clone(), Value::Object(out), - )) + ) + .preserve_expiry_from(source)) } fn run(source: &SourceRecord) -> Option { @@ -205,7 +207,8 @@ fn run(source: &SourceRecord) -> Option { RUNS, source.id.clone(), Value::Object(out), - )) + ) + .preserve_expiry_from(source)) } /// `cron_counters` holds two documents: the run counter and the admission @@ -229,13 +232,17 @@ fn counter_record(source: &SourceRecord) -> Option { out.insert("limit".into(), limit.clone()); } out.insert("members".into(), Value::Array(members)); - Some(TargetRecord::new(ADMISSION, LEDGER, Value::Object(out))) + Some( + TargetRecord::new(ADMISSION, LEDGER, Value::Object(out)) + .preserve_expiry_from(source), + ) } "runs" => Some(TargetRecord::new( ADMISSION, STAGED_COUNTER, json!({ "next": source.body.get("next")?.as_i64()? }), - )), + ) + .preserve_expiry_from(source)), _ => None, } } From 697e0b2caaa48bcf0699db27ed02de5cc688b076 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:06:47 +0300 Subject: [PATCH 49/64] fix(store): preserve record expiry during migration Migrated records now carry their original expiry timestamp over to the target record instead of dropping it, so records that were set to expire keep their expiry after the v1 to v2 migration. Auto-committed-on: dragonfly --- crates/tinyflows-drivers/src/store/migrate.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/tinyflows-drivers/src/store/migrate.rs b/crates/tinyflows-drivers/src/store/migrate.rs index e45face0..c216751b 100644 --- a/crates/tinyflows-drivers/src/store/migrate.rs +++ b/crates/tinyflows-drivers/src/store/migrate.rs @@ -82,7 +82,7 @@ fn key(id: &str, fallback: &str) -> String { /// The record carried unchanged under its v1 id. fn carried(collection: &str, source: &SourceRecord) -> TargetRecord { - TargetRecord::new(collection, source.id.clone(), source.body.clone()) + TargetRecord::new(collection, source.id.clone(), source.body.clone()).preserve_expiry_from(source) } fn str_field<'a>(map: &'a Map, field: &str) -> Option<&'a str> { @@ -101,7 +101,8 @@ fn definition(source: &SourceRecord) -> Option { DEFINITIONS, key(&id, &source.id), to_stored(Value::Object(record)), - )) + ) + .preserve_expiry_from(source)) } fn revision(source: &SourceRecord) -> Option { @@ -133,7 +134,8 @@ fn revision(source: &SourceRecord) -> Option { REVISIONS, source.id.clone(), Value::Object(body), - )) + ) + .preserve_expiry_from(source)) } fn run(source: &SourceRecord) -> Option { @@ -161,7 +163,8 @@ fn run(source: &SourceRecord) -> Option { RUNS, key(&id, &source.id), Value::Object(body), - )) + ) + .preserve_expiry_from(source)) } fn journal(source: &SourceRecord) -> Option { @@ -179,7 +182,8 @@ fn journal(source: &SourceRecord) -> Option { NOTES, key(workflow_id, &source.id), json!({ "workflow_id": workflow_id, "notes": notes }), - )) + ) + .preserve_expiry_from(source)) } fn proposal(source: &SourceRecord) -> Option { @@ -193,7 +197,8 @@ fn proposal(source: &SourceRecord) -> Option { PROPOSALS, key(&id, &source.id), to_stored(Value::Object(proposal)), - )) + ) + .preserve_expiry_from(source)) } fn quarantine(source: &SourceRecord) -> Option { From 5c2de121e0f4fa2c5b6d16561ccce0b7cc9a1525 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 21 Oct 2026 00:27:47 +0300 Subject: [PATCH 50/64] fix(storage): preserve migrated chronology and expiry --- .../tinyflows-adaptive/src/drivers/migrate.rs | 75 +++++++++++-------- .../src/drivers/migrate_tests.rs | 2 +- .../src/catalog/migrate_tests.rs | 5 +- .../tinyflows-drivers/src/schedule/codec.rs | 9 ++- crates/tinyflows-drivers/src/schedule/jobs.rs | 6 +- .../tinyflows-drivers/src/schedule/migrate.rs | 28 +++---- .../src/schedule/migrate_tests.rs | 33 +++++++- crates/tinyflows-drivers/src/schedule/runs.rs | 2 +- .../src/schedule/runs_tests.rs | 30 ++++++++ crates/tinyflows-drivers/src/store/migrate.rs | 55 +++++++------- .../src/catalog_service_migration_tests.rs | 7 +- .../storage_migration_artifact_tests.rs | 8 +- crates/tinyflows-storage/src/schema_tests.rs | 11 +-- 13 files changed, 178 insertions(+), 93 deletions(-) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate.rs b/crates/tinyflows-adaptive/src/drivers/migrate.rs index 48c3e508..44b23ac1 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate.rs @@ -97,6 +97,7 @@ fn key(parts: &[&str], fallback: &str) -> String { fn carried(collection: &str, source: &SourceRecord) -> TargetRecord { TargetRecord::new(collection, source.id.clone(), source.body.clone()) + .preserve_expiry_from(source) } fn row(source: &SourceRecord) -> Option { @@ -104,7 +105,7 @@ fn row(source: &SourceRecord) -> Option { return Some(carried(ROWS, source)); }; let body = row_body(rec, text(source, "scope_key"), seq(source)); - Some(TargetRecord::new(ROWS, source.id.clone(), body)) + Some(TargetRecord::new(ROWS, source.id.clone(), body).preserve_expiry_from(source)) } fn lesson(source: &SourceRecord) -> Option { @@ -117,7 +118,7 @@ fn lesson(source: &SourceRecord) -> Option { } } let body = lesson_body(rec, text(source, "scope_key"), seq(source)); - Some(TargetRecord::new(LESSONS, source.id.clone(), body)) + Some(TargetRecord::new(LESSONS, source.id.clone(), body).preserve_expiry_from(source)) } fn evidence(source: &SourceRecord) -> Option { @@ -125,11 +126,14 @@ fn evidence(source: &SourceRecord) -> Option { if lesson.is_empty() || row.is_empty() { return Some(carried(EVIDENCE, source)); } - Some(TargetRecord::new( - EVIDENCE, - key(&[lesson, row], &source.id), - source.body.clone(), - )) + Some( + TargetRecord::new( + EVIDENCE, + key(&[lesson, row], &source.id), + source.body.clone(), + ) + .preserve_expiry_from(source), + ) } fn score(source: &SourceRecord) -> Option { @@ -137,11 +141,14 @@ fn score(source: &SourceRecord) -> Option { if workflow.is_empty() { return Some(carried(SCORES, source)); } - Some(TargetRecord::new( - SCORES, - key(&[scope, workflow], &source.id), - source.body.clone(), - )) + Some( + TargetRecord::new( + SCORES, + key(&[scope, workflow], &source.id), + source.body.clone(), + ) + .preserve_expiry_from(source), + ) } fn variant(source: &SourceRecord) -> Option { @@ -149,11 +156,14 @@ fn variant(source: &SourceRecord) -> Option { if variant.is_empty() { return Some(carried(VARIANTS, source)); } - Some(TargetRecord::new( - VARIANTS, - key(&[scope, variant], &source.id), - source.body.clone(), - )) + Some( + TargetRecord::new( + VARIANTS, + key(&[scope, variant], &source.id), + source.body.clone(), + ) + .preserve_expiry_from(source), + ) } fn episode(source: &SourceRecord) -> Option { @@ -167,11 +177,10 @@ fn episode(source: &SourceRecord) -> Option { .unwrap_or_default() .to_owned(); let body = episode_body(rec, scope); - Some(TargetRecord::new( - EPISODES, - key(&[scope, &id], &source.id), - body, - )) + Some( + TargetRecord::new(EPISODES, key(&[scope, &id], &source.id), body) + .preserve_expiry_from(source), + ) } fn steps(source: &SourceRecord) -> Option { @@ -184,11 +193,14 @@ fn steps(source: &SourceRecord) -> Option { seq(source), ); let body = step_body(rec, scope, row_id, n); - Some(TargetRecord::new( - STEPS, - key(&[scope, row_id, &format!("{n:08}")], &source.id), - body, - )) + Some( + TargetRecord::new( + STEPS, + key(&[scope, row_id, &format!("{n:08}")], &source.id), + body, + ) + .preserve_expiry_from(source), + ) } fn workflow(source: &SourceRecord) -> Option { @@ -202,11 +214,10 @@ fn workflow(source: &SourceRecord) -> Option { .unwrap_or_default() .to_owned(); let target_key = key(&[scope, &id], &source.id); - Some(TargetRecord::new( - WORKFLOWS, - target_key, - workflow_body(rec, scope), - )) + Some( + TargetRecord::new(WORKFLOWS, target_key, workflow_body(rec, scope)) + .preserve_expiry_from(source), + ) } /// Move the v1 id sequences (`adaptive_counters`: `{seq}` per sequence name) diff --git a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs index fe7aef69..dded46de 100644 --- a/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs +++ b/crates/tinyflows-adaptive/src/drivers/migrate_tests.rs @@ -105,7 +105,7 @@ async fn a_v1_row_migrates_to_a_flat_body_with_an_ms_time() { assert_eq!(stored.doc["at_ms"], 1_767_225_602_000_i64); assert_eq!(stored.doc["satisfied"], true); assert!(stored.doc.get("record").is_none() && stored.doc.get("at").is_none()); - assert_eq!(stored.meta.created_ms, 1_767_225_602_000_i64); + assert!(stored.meta.created_ms > 0, "backend owns record metadata"); let ledger = DriverLedger::new(docs(&storage)).for_tenant("t"); let rows = ledger.rows("ep").await.unwrap(); assert_eq!(rows[0].id, "ldg_00000007"); diff --git a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs index aa8b6a41..9e1a1ba2 100644 --- a/crates/tinyflows-drivers/src/catalog/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/catalog/migrate_tests.rs @@ -149,10 +149,7 @@ async fn v1_catalog_records_migrate_to_bare_ms_bodies_and_read_back() { assert_eq!(def.doc["graph"], graph(), "the graph string is parsed"); assert_eq!(def.doc["created_ms"], 1_772_600_767_250_i64); assert_eq!(def.doc["last_run_ms"], 1_772_600_767_250_i64); - assert_eq!( - def.meta.created_ms, 1_772_600_767_250_i64, - "history keeps its time" - ); + assert!(def.meta.created_ms > 0, "backend owns record metadata"); for gone in [ "created_ns", "created_at", diff --git a/crates/tinyflows-drivers/src/schedule/codec.rs b/crates/tinyflows-drivers/src/schedule/codec.rs index bf8922fc..9fd08ebe 100644 --- a/crates/tinyflows-drivers/src/schedule/codec.rs +++ b/crates/tinyflows-drivers/src/schedule/codec.rs @@ -165,6 +165,13 @@ pub(super) fn set_last_run( doc.insert("last_output".into(), json!(output)); } +/// The creation identity used for run fencing and duplicate ranking. +/// Migrated jobs carry their original creation time in the body because the +/// module document port cannot write host-owned metadata. +pub(super) fn job_created_ms(doc: &Value, metadata_created_ms: i64) -> Result { + Ok(integer(doc, "created_ms")?.unwrap_or(metadata_created_ms)) +} + /// The job a stored document holds. Imported jobs retain their original /// creation time in the body because module document ports cannot set the /// host-owned record metadata; ordinary jobs still use backend metadata. @@ -217,7 +224,7 @@ pub(super) fn doc_to_job(stored: &Versioned) -> Result { enabled: flag("enabled")?, delivery, delete_after_run: flag("delete_after_run")?, - created_at: from_ms(integer(doc, "created_ms")?.unwrap_or(stored.meta.created_ms))?, + created_at: from_ms(job_created_ms(doc, stored.meta.created_ms)?)?, next_run: from_ms( integer(doc, "next_run_ms")? .ok_or_else(|| anyhow!("cron store: document has no `next_run_ms`"))?, diff --git a/crates/tinyflows-drivers/src/schedule/jobs.rs b/crates/tinyflows-drivers/src/schedule/jobs.rs index 8e74e1d5..73cdf9de 100644 --- a/crates/tinyflows-drivers/src/schedule/jobs.rs +++ b/crates/tinyflows-drivers/src/schedule/jobs.rs @@ -14,7 +14,9 @@ use tinyflows_schedule::{ use tinyflows_storage::{DocumentStoreExt, Filter, Query, Sort, Versioned, WriteResult}; use uuid::Uuid; -use super::codec::{INCARNATION, doc_to_job, incarnation, job_to_doc, set_next_run, text}; +use super::codec::{ + INCARNATION, doc_to_job, incarnation, job_created_ms, job_to_doc, set_next_run, text, +}; use super::patch::apply_patch; use super::{CAS_ATTEMPTS, CronDocuments, JOBS, RUNS, job_not_found, new_id, storage_error}; @@ -316,7 +318,7 @@ impl CronDocuments { .count(RUNS, &runs_of(&stored.id, &stored.doc)) .await .map_err(storage_error)?; - let created = stored.meta.created_ms; + let created = job_created_ms(&stored.doc, stored.meta.created_ms)?; ranked.push((std::cmp::Reverse(runs), created, stored.id.clone(), stored)); } ranked.sort_by(|a, b| (&a.0, &a.1, &a.2).cmp(&(&b.0, &b.1, &b.2))); diff --git a/crates/tinyflows-drivers/src/schedule/migrate.rs b/crates/tinyflows-drivers/src/schedule/migrate.rs index ca8ebdad..0a1dffa9 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate.rs @@ -167,12 +167,9 @@ fn job(source: &SourceRecord) -> Option { .or_else(|| body.get("created_ms").and_then(Value::as_i64)) .unwrap_or(source.meta.created_ms); out.insert("created_ms".into(), created.into()); - Some(TargetRecord::new( - JOBS, - source.id.clone(), - Value::Object(out), + Some( + TargetRecord::new(JOBS, source.id.clone(), Value::Object(out)).preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn run(source: &SourceRecord) -> Option { @@ -203,12 +200,9 @@ fn run(source: &SourceRecord) -> Option { "finished_ms".into(), ms("finished_at", "finished_ns")?.into(), ); - Some(TargetRecord::new( - RUNS, - source.id.clone(), - Value::Object(out), + Some( + TargetRecord::new(RUNS, source.id.clone(), Value::Object(out)).preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } /// `cron_counters` holds two documents: the run counter and the admission @@ -237,12 +231,14 @@ fn counter_record(source: &SourceRecord) -> Option { .preserve_expiry_from(source), ) } - "runs" => Some(TargetRecord::new( - ADMISSION, - STAGED_COUNTER, - json!({ "next": source.body.get("next")?.as_i64()? }), - ) - .preserve_expiry_from(source)), + "runs" => Some( + TargetRecord::new( + ADMISSION, + STAGED_COUNTER, + json!({ "next": source.body.get("next")?.as_i64()? }), + ) + .preserve_expiry_from(source), + ), _ => None, } } diff --git a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs index 78c49dc2..d68774ab 100644 --- a/crates/tinyflows-drivers/src/schedule/migrate_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/migrate_tests.rs @@ -1,7 +1,7 @@ use super::*; use chrono::TimeZone; use serde_json::json; -use tinyflows_schedule::Schedule; +use tinyflows_schedule::{Schedule, runtime::RunRecord}; use tinyflows_storage::{ DocumentStoreExt, MigrateOptions, Precondition, Query, Scope, Storage, StorageBackend, }; @@ -175,6 +175,37 @@ async fn v1_jobs_runs_and_the_ledger_migrate_and_read_back() { ); } +#[tokio::test] +async fn a_claimed_run_accepts_a_migrated_jobs_body_creation_time() { + let storage = scoped(); + seed( + &storage, + "cron_jobs", + "migrated-job", + v1_job(json!({"kind": "cron", "expr": "0 9 * * *", "tz": null, "active_hours": null})), + ) + .await; + migrate(&storage).await; + + let store = CronDocuments::new(std::sync::Arc::clone(storage.documents())); + let created_at = chrono::DateTime::from_timestamp_millis(1_772_600_767_123_i64).unwrap(); + let now = chrono::Utc::now(); + store + .record_claimed_run(&RunRecord { + run_id: "migration-run".into(), + job_id: "migrated-job".into(), + job_created_at: created_at, + started_at: now, + finished_at: now, + status: "ok".into(), + output: "done".into(), + delivery_status: None, + }) + .await + .expect("a migration's body creation time is the job identity"); + assert_eq!(store.list_runs("migrated-job", 10).await.unwrap().len(), 1); +} + #[tokio::test] async fn a_v1_claims_queued_placeholder_run_is_kept_on_the_job_and_removed_by_the_next_claim() { let storage = scoped(); diff --git a/crates/tinyflows-drivers/src/schedule/runs.rs b/crates/tinyflows-drivers/src/schedule/runs.rs index 7d2106e6..25e23fd7 100644 --- a/crates/tinyflows-drivers/src/schedule/runs.rs +++ b/crates/tinyflows-drivers/src/schedule/runs.rs @@ -200,7 +200,7 @@ impl CronDocuments { let stored = self.job_doc(job_id).await?; if let Some(expected) = expected_creation { anyhow::ensure!( - from_ms(stored.meta.created_ms)? == expected, + from_ms(codec::job_created_ms(&stored.doc, stored.meta.created_ms)?)? == expected, "cron run belongs to a removed job creation" ); } diff --git a/crates/tinyflows-drivers/src/schedule/runs_tests.rs b/crates/tinyflows-drivers/src/schedule/runs_tests.rs index 7a946556..a28d8b3e 100644 --- a/crates/tinyflows-drivers/src/schedule/runs_tests.rs +++ b/crates/tinyflows-drivers/src/schedule/runs_tests.rs @@ -308,6 +308,36 @@ async fn dedup_ties_go_to_the_earliest_created() { assert_eq!(left[0].id, first.id); } +#[tokio::test] +async fn dedup_ranks_migrated_jobs_by_their_body_creation_time() { + let store = store(); + let first = store + .add_shell_job(Some("imported-dup".into()), daily(), "a") + .await + .unwrap(); + let second = store + .add_shell_job(Some("imported-dup".into()), daily(), "b") + .await + .unwrap(); + + // Migration/import chronology lives in the body while backend metadata + // reflects when these records were inserted. + for (id, created_ms) in [(&first.id, 2_000_i64), (&second.id, 1_000_i64)] { + let stored = store.docs.get(JOBS, id).await.unwrap().unwrap(); + let mut doc = stored.doc.as_object().cloned().unwrap(); + doc.insert("created_ms".into(), json!(created_ms)); + store + .docs + .put(JOBS, id, Value::Object(doc), stored.unchanged()) + .await + .unwrap(); + } + + assert_eq!(store.dedup_named_jobs().await.unwrap(), 1); + assert_eq!(store.get_job(&second.id).await.unwrap().id, second.id); + assert!(store.get_job(&first.id).await.is_err()); +} + #[tokio::test] async fn a_run_for_a_missing_job_is_refused() { let store = store(); diff --git a/crates/tinyflows-drivers/src/store/migrate.rs b/crates/tinyflows-drivers/src/store/migrate.rs index c216751b..2802c742 100644 --- a/crates/tinyflows-drivers/src/store/migrate.rs +++ b/crates/tinyflows-drivers/src/store/migrate.rs @@ -82,7 +82,8 @@ fn key(id: &str, fallback: &str) -> String { /// The record carried unchanged under its v1 id. fn carried(collection: &str, source: &SourceRecord) -> TargetRecord { - TargetRecord::new(collection, source.id.clone(), source.body.clone()).preserve_expiry_from(source) + TargetRecord::new(collection, source.id.clone(), source.body.clone()) + .preserve_expiry_from(source) } fn str_field<'a>(map: &'a Map, field: &str) -> Option<&'a str> { @@ -97,12 +98,14 @@ fn definition(source: &SourceRecord) -> Option { return Some(carried(DEFINITIONS, source)); }; record.remove("source_path"); - Some(TargetRecord::new( - DEFINITIONS, - key(&id, &source.id), - to_stored(Value::Object(record)), + Some( + TargetRecord::new( + DEFINITIONS, + key(&id, &source.id), + to_stored(Value::Object(record)), + ) + .preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn revision(source: &SourceRecord) -> Option { @@ -130,12 +133,10 @@ fn revision(source: &SourceRecord) -> Option { }; body.insert("workflow_id".into(), json!(workflow_id)); body.insert("superseded_ms".into(), json!(superseded)); - Some(TargetRecord::new( - REVISIONS, - source.id.clone(), - Value::Object(body), + Some( + TargetRecord::new(REVISIONS, source.id.clone(), Value::Object(body)) + .preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn run(source: &SourceRecord) -> Option { @@ -159,12 +160,10 @@ fn run(source: &SourceRecord) -> Option { return None; }; body.insert("settled".into(), json!(settled)); - Some(TargetRecord::new( - RUNS, - key(&id, &source.id), - Value::Object(body), + Some( + TargetRecord::new(RUNS, key(&id, &source.id), Value::Object(body)) + .preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn journal(source: &SourceRecord) -> Option { @@ -178,12 +177,14 @@ fn journal(source: &SourceRecord) -> Option { return Some(carried(NOTES, source)); }; let notes: Vec = notes.into_iter().map(to_stored).collect(); - Some(TargetRecord::new( - NOTES, - key(workflow_id, &source.id), - json!({ "workflow_id": workflow_id, "notes": notes }), + Some( + TargetRecord::new( + NOTES, + key(workflow_id, &source.id), + json!({ "workflow_id": workflow_id, "notes": notes }), + ) + .preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn proposal(source: &SourceRecord) -> Option { @@ -193,12 +194,14 @@ fn proposal(source: &SourceRecord) -> Option { let Some(id) = str_field(&proposal, "id").map(str::to_owned) else { return Some(carried(PROPOSALS, source)); }; - Some(TargetRecord::new( - PROPOSALS, - key(&id, &source.id), - to_stored(Value::Object(proposal)), + Some( + TargetRecord::new( + PROPOSALS, + key(&id, &source.id), + to_stored(Value::Object(proposal)), + ) + .preserve_expiry_from(source), ) - .preserve_expiry_from(source)) } fn quarantine(source: &SourceRecord) -> Option { diff --git a/crates/tinyflows-module/src/catalog_service_migration_tests.rs b/crates/tinyflows-module/src/catalog_service_migration_tests.rs index a49bbca0..74eada5e 100644 --- a/crates/tinyflows-module/src/catalog_service_migration_tests.rs +++ b/crates/tinyflows-module/src/catalog_service_migration_tests.rs @@ -92,14 +92,17 @@ async fn explicit_catalog_migration_preserves_records_and_imports_legacy_state() .unwrap(); assert_eq!(migrated.doc["name"], "Legacy"); assert_eq!(migrated.doc["created_ms"], created_ms); - assert_eq!(migrated.meta.created_ms, created_ms); + assert!(migrated.meta.created_ms > 0, "backend owns record metadata"); let migrated_revision = docs .get("flows.revisions", "legacy-revision") .await .unwrap() .unwrap(); assert_eq!(migrated_revision.doc["created_ms"], revision_created_ms); - assert_eq!(migrated_revision.meta.created_ms, revision_created_ms); + assert!( + migrated_revision.meta.created_ms > 0, + "backend owns record metadata" + ); assert!( docs.get("flows_definitions", "legacy-flow") .await diff --git a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs index 66286b9c..a00fd0df 100644 --- a/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs +++ b/crates/tinyflows-module/tests/support/storage_migration_artifact_tests.rs @@ -184,7 +184,8 @@ async fn published_module_migrates_legacy_schedule_records_through_owned_bus_cal chrono::DateTime::parse_from_rfc3339(legacy["doc"]["created_at"].as_str().unwrap()) .unwrap() .timestamp_millis(); - assert_eq!(migrated.meta.created_ms, legacy_created_ms); + assert_eq!(migrated.doc["created_ms"], legacy_created_ms); + assert!(migrated.meta.created_ms > 0, "backend owns record metadata"); let jobs = CronDocuments::new(docs).list_jobs().await.unwrap(); assert_eq!(jobs.len(), 1, "the legacy job is visible after migration"); assert_eq!(jobs[0].id, legacy_id); @@ -401,7 +402,10 @@ async fn published_module_migrates_legacy_catalog_records_through_owned_bus_call .unwrap() .unwrap(); assert_eq!(migrated_revision.doc["created_ms"], revision_created_ms); - assert_eq!(migrated_revision.meta.created_ms, revision_created_ms); + assert!( + migrated_revision.meta.created_ms > 0, + "backend owns record metadata" + ); let catalog_state = tinyflows_storage::Key::of(["flows", "kv:legacy-flow", "k"]).unwrap(); let state = tinyflows_storage::Key::of(["flows", "state", "state-key"]).unwrap(); assert_eq!( diff --git a/crates/tinyflows-storage/src/schema_tests.rs b/crates/tinyflows-storage/src/schema_tests.rs index f2d05bb5..632a4390 100644 --- a/crates/tinyflows-storage/src/schema_tests.rs +++ b/crates/tinyflows-storage/src/schema_tests.rs @@ -232,11 +232,12 @@ async fn renamed_migration_targets_keep_their_source_absolute_expiry() { .expect("read renamed target") .expect("renamed target exists"); assert_eq!(target.meta.expires_ms, Some(expiry)); - assert!(docs - .get("legacy.items", "old-id") - .await - .expect("read source") - .is_none()); + assert!( + docs.get("legacy.items", "old-id") + .await + .expect("read source") + .is_none() + ); } #[tokio::test] From 7d463fbc67ecabbffa09c2c8e4fb43e25add1779 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 24 Oct 2026 20:08:00 +0000 Subject: [PATCH 51/64] fix(module): align TinyBus dependency with 0.3 Co-authored-by: Medulla --- Cargo.lock | 173 ++++++++++++++--------------- Cargo.toml | 4 +- crates/tinyflows-module/Cargo.toml | 4 +- 3 files changed, 87 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25f97e01..8856b402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,15 +23,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -329,12 +320,6 @@ dependencies = [ "winnow 0.7.15", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" - [[package]] name = "crypto-common" version = "0.1.7" @@ -360,17 +345,6 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" -[[package]] -name = "derive_arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - [[package]] name = "digest" version = "0.10.7" @@ -428,7 +402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -789,14 +763,37 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_casemap" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a6f143e9349267b80ae071a6ec25d4797c45756d8b2671a8b24b7b25b81ab94" +dependencies = [ + "icu_casemap_data", + "icu_collections", + "icu_locale_core", + "icu_properties", + "icu_provider", + "potential_utf", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_casemap_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71a5c321208537711696af55bc2ab5dad939e4f6e0aef92dd6a2555deceedf3a" + [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", + "serde", "utf8_iter", "yoke", "zerofrom", @@ -805,12 +802,13 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", @@ -818,9 +816,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -832,15 +830,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -853,18 +851,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1266,6 +1266,8 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -1382,7 +1384,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1485,6 +1487,12 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rawzip" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9999039f74cd73c03158cddfe4362991703c74efe5ce5c8073026f80a5e48c" + [[package]] name = "regex-automata" version = "0.4.18" @@ -1598,7 +1606,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1657,7 +1665,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1982,7 +1990,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2007,12 +2015,14 @@ dependencies = [ [[package]] name = "tinybus" -version = "0.1.5" -source = "git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336#eb5cf727071a6eb12f9efa3db8ec59e54039f336" +version = "0.3.12" +source = "git+https://github.com/senamakel/tinybus?rev=34d3bd6dda099e54cee11a2f96f28a21da9e0759#34d3bd6dda099e54cee11a2f96f28a21da9e0759" dependencies = [ "async-trait", "flate2", + "icu_casemap", "libc", + "rawzip", "serde", "serde_json", "sha2 0.10.9", @@ -2023,14 +2033,14 @@ dependencies = [ "tokio", "toml", "tracing", + "unicode-normalization", "ureq", - "zip", ] [[package]] name = "tinybus-macros" -version = "0.1.5" -source = "git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336#eb5cf727071a6eb12f9efa3db8ec59e54039f336" +version = "0.3.12" +source = "git+https://github.com/senamakel/tinybus?rev=34d3bd6dda099e54cee11a2f96f28a21da9e0759#34d3bd6dda099e54cee11a2f96f28a21da9e0759" dependencies = [ "proc-macro2", "quote", @@ -2039,8 +2049,8 @@ dependencies = [ [[package]] name = "tinybus-module" -version = "0.1.5" -source = "git+https://github.com/tinyhumansai/tinybus?rev=eb5cf727071a6eb12f9efa3db8ec59e54039f336#eb5cf727071a6eb12f9efa3db8ec59e54039f336" +version = "0.3.12" +source = "git+https://github.com/senamakel/tinybus?rev=34d3bd6dda099e54cee11a2f96f28a21da9e0759#34d3bd6dda099e54cee11a2f96f28a21da9e0759" dependencies = [ "async-trait", "serde", @@ -2247,11 +2257,12 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -2489,6 +2500,15 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -2715,7 +2735,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2901,9 +2921,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -2978,14 +2998,16 @@ dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -2993,32 +3015,15 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", "syn 3.0.6", ] -[[package]] -name = "zip" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap", - "memchr", - "thiserror", - "zopfli", -] - [[package]] name = "zlib-rs" version = "0.6.8" @@ -3031,18 +3036,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 657d4afd..19a6a22b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,8 @@ exclude = ["worktrees", "vendor/tinybus"] # Pin unpublished module APIs to a source available in non-recursive clones and archives. [patch.crates-io] -tinybus = { git = "https://github.com/tinyhumansai/tinybus", rev = "eb5cf727071a6eb12f9efa3db8ec59e54039f336" } -tinybus-module = { git = "https://github.com/tinyhumansai/tinybus", rev = "eb5cf727071a6eb12f9efa3db8ec59e54039f336" } +tinybus = { git = "https://github.com/senamakel/tinybus", rev = "34d3bd6dda099e54cee11a2f96f28a21da9e0759" } +tinybus-module = { git = "https://github.com/senamakel/tinybus", rev = "34d3bd6dda099e54cee11a2f96f28a21da9e0759" } [workspace.package] edition = "2024" diff --git a/crates/tinyflows-module/Cargo.toml b/crates/tinyflows-module/Cargo.toml index 33b2500a..e560db08 100644 --- a/crates/tinyflows-module/Cargo.toml +++ b/crates/tinyflows-module/Cargo.toml @@ -28,8 +28,8 @@ tinyflows = { workspace = true, features = ["mock"] } tinyflows-bus = { path = "../tinyflows-bus" } tinyflows-copilot = { path = "../tinyflows-copilot" } tinytools = { git = "https://github.com/tinyhumansai/tinytools", rev = "643df90ba25f93187249fa99f6789f9aa7f8bbc8" } -tinybus = { version = ">=0.1.3, <0.2", default-features = false, features = ["macros", "modules"] } -tinybus-module = ">=0.1.3, <0.2" +tinybus = { version = ">=0.3.12, <0.4", default-features = false, features = ["macros", "modules"] } +tinybus-module = ">=0.3.12, <0.4" serde_json.workspace = true [features] From 6c80292adc56457639533b3a0ca208c4a2a4aa88 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 04:32:21 +0000 Subject: [PATCH 52/64] chore(deps): downgrade uuid and align getrandom version Pinned uuid back to 1.26.1 and switched tempfile's getrandom dependency to 0.3.4 so the lockfile resolves a single compatible getrandom version. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b9539c0..bdb8491a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,7 +2025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2759,9 +2759,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.28.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", From 291deae2b7086d49ad827cdb7ae061f8758d1e80 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 04:36:54 +0000 Subject: [PATCH 53/64] chore(deps): bump tinystoragedrivers to 2.1.2 and tinybus to 0.1.6 Update the tinystoragedrivers git dependencies from the v2.0.1 tag to revision 2d23a6a (2.1.2) and tinybus from 0.1.5 to 0.1.6, refreshing Cargo.lock accordingly. The new revision pulls in nix and drops the unused arbitrary and crossbeam-utils crates. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- Cargo.lock | 135 ++++++++++++--------------- Cargo.toml | 2 +- crates/tinyflows-adaptive/Cargo.toml | 10 +- crates/tinyflows-drivers/Cargo.toml | 6 +- crates/tinyflows-module/Cargo.toml | 6 +- crates/tinyflows-storage/Cargo.toml | 10 +- 6 files changed, 76 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdb8491a..f1886759 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,15 +23,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -329,12 +320,6 @@ dependencies = [ "winnow 0.7.15", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" - [[package]] name = "crypto-common" version = "0.1.7" @@ -360,17 +345,6 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" -[[package]] -name = "derive_arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - [[package]] name = "digest" version = "0.10.7" @@ -1167,6 +1141,18 @@ dependencies = [ "pxfm", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -2053,8 +2039,8 @@ dependencies = [ [[package]] name = "tinybus" -version = "0.1.5" -source = "git+https://github.com/senamakel/tinybus?rev=d5b949be8ea56eb5a17289a09aa28e8c09bd88b7#d5b949be8ea56eb5a17289a09aa28e8c09bd88b7" +version = "0.1.6" +source = "git+https://github.com/senamakel/tinybus?rev=0de2d283f35cbbff08efad6fbe05537ffb74611b#0de2d283f35cbbff08efad6fbe05537ffb74611b" dependencies = [ "async-trait", "flate2", @@ -2280,7 +2266,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "tinybus 0.1.5", + "tinybus 0.1.6", "tinystoragedrivers-api", "tinystoragedrivers-busproxy", "tinystoragedrivers-testkit", @@ -2291,8 +2277,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-api" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "bytes", @@ -2305,8 +2291,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-blob" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "bytes", @@ -2319,26 +2305,26 @@ dependencies = [ [[package]] name = "tinystoragedrivers-bus" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "serde", "serde_json", - "tinybus 0.1.5", + "tinybus 0.1.6", "tinystoragedrivers-types", ] [[package]] name = "tinystoragedrivers-busproxy" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "bytes", "futures-core", "serde", "serde_json", - "tinybus 0.1.5", + "tinybus 0.1.6", "tinystoragedrivers-api", "tinystoragedrivers-bus", "tinystoragedrivers-types", @@ -2348,25 +2334,27 @@ dependencies = [ [[package]] name = "tinystoragedrivers-connect" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "rusqlite", + "serde_json", "tinystoragedrivers-types", "tokio", + "zstd", ] [[package]] name = "tinystoragedrivers-document" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "data-encoding", "rusqlite", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tinystoragedrivers-api", "tinystoragedrivers-connect", "tinystoragedrivers-types", @@ -2376,8 +2364,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-kv" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "serde", @@ -2388,14 +2376,14 @@ dependencies = [ [[package]] name = "tinystoragedrivers-queue" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "data-encoding", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tinystoragedrivers-api", "tinystoragedrivers-types", "tokio", @@ -2404,8 +2392,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-secrets" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "tinystoragedrivers-api", @@ -2415,8 +2403,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-testkit" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "async-trait", "bytes", @@ -2436,8 +2424,8 @@ dependencies = [ [[package]] name = "tinystoragedrivers-types" -version = "2.0.1" -source = "git+https://github.com/senamakel/tinystoragedrivers?tag=v2.0.1#e3224a8262193e7466b230a59ef1c4769e44bf02" +version = "2.1.2" +source = "git+https://github.com/senamakel/tinystoragedrivers?rev=2d23a6a6009fa52d1e7aa18de589bdd109a54b6a#2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" dependencies = [ "serde", "serde_json", @@ -2661,6 +2649,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-path" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce898823d8cd0f028662d2f349694b1af0a5cc9be83f1d213a00d3147445e1e" + [[package]] name = "typenum" version = "1.20.1" @@ -3215,17 +3209,18 @@ dependencies = [ [[package]] name = "zip" -version = "2.6.1" +version = "9.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +checksum = "ec2e3161de27091f18a831d9494e9c53378a34833af5c1dda35ad85849da18d6" dependencies = [ - "arbitrary", "crc32fast", - "crossbeam-utils", "flate2", + "getrandom 0.4.3", "indexmap", "memchr", - "zopfli", + "nix", + "typed-path", + "windows-sys 0.61.2", ] [[package]] @@ -3240,32 +3235,20 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" -[[package]] -name = "zopfli" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 57284382..cb362159 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ tinybus = { git = "https://github.com/senamakel/tinybus", rev = "34d3bd6dda099e5 tinybus-module = { git = "https://github.com/senamakel/tinybus", rev = "34d3bd6dda099e54cee11a2f96f28a21da9e0759" } [patch."https://github.com/tinyhumansai/tinybus"] -tinybus = { git = "https://github.com/senamakel/tinybus", rev = "d5b949be8ea56eb5a17289a09aa28e8c09bd88b7" } +tinybus = { git = "https://github.com/senamakel/tinybus", rev = "0de2d283f35cbbff08efad6fbe05537ffb74611b" } [workspace.package] edition = "2024" diff --git a/crates/tinyflows-adaptive/Cargo.toml b/crates/tinyflows-adaptive/Cargo.toml index 2e3cb57e..94b0d536 100644 --- a/crates/tinyflows-adaptive/Cargo.toml +++ b/crates/tinyflows-adaptive/Cargo.toml @@ -19,8 +19,8 @@ thiserror = "2" # tag so a host that vendors tinystoragedrivers once can `[patch]` it; chrono # turns the RFC 3339 instants of the records into the stored `*_ms`; tokio's # `OnceCell` for one-time collection setup. -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1", optional = true } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1", optional = true } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a", optional = true } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a", optional = true } tinyflows-storage = { path = "../tinyflows-storage", optional = true } chrono = { version = "0.4", default-features = false, features = ["std"], optional = true } tracing = { version = "0.1", default-features = false, features = ["std"], optional = true } @@ -41,9 +41,9 @@ tinyflows = { path = "../tinyflows", features = ["store", "mock"] } # The engine's `WorkflowStore` on an in-memory storage backend, for tests and # the eval example. tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["store"] } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } [lints.rust] diff --git a/crates/tinyflows-drivers/Cargo.toml b/crates/tinyflows-drivers/Cargo.toml index 5cb48bf7..ae83bc08 100644 --- a/crates/tinyflows-drivers/Cargo.toml +++ b/crates/tinyflows-drivers/Cargo.toml @@ -51,8 +51,8 @@ store = [ test-fixtures = ["catalog"] [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } tinyflows-storage = { path = "../tinyflows-storage" } serde = { workspace = true } serde_json = { workspace = true } @@ -72,4 +72,4 @@ uuid = { workspace = true, optional = true } [dev-dependencies] async-trait = { workspace = true } tokio = { workspace = true } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } diff --git a/crates/tinyflows-module/Cargo.toml b/crates/tinyflows-module/Cargo.toml index 64d62a55..0b37ae9e 100644 --- a/crates/tinyflows-module/Cargo.toml +++ b/crates/tinyflows-module/Cargo.toml @@ -14,8 +14,8 @@ crate-type = ["rlib", "cdylib"] [dependencies] tinyflows-drivers = { path = "../tinyflows-drivers", default-features = false, features = ["catalog", "engine", "schedule"] } tinyflows-catalog = { path = "../tinyflows-catalog" } -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } tinyflows-storage = { path = "../tinyflows-storage" } tinyflows-schedule = { path = "../tinyflows-schedule" } anyhow = "1" @@ -42,4 +42,4 @@ static-link = [] [dev-dependencies] tempfile = "3" tokio = { workspace = true, features = ["test-util"] } -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index 737782a4..a565e2ea 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -9,8 +9,8 @@ publish = false description = "TinyFlows-owned generic document persistence helpers" [dependencies] -tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } +tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } serde = { workspace = true } serde_json = { workspace = true } async-trait = { workspace = true } @@ -20,6 +20,6 @@ ulid = "1" data-encoding = "2" [dev-dependencies] -tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.0.1", tag = "v2.0.1" } -tinybus-sdk = { package = "tinybus", git = "https://github.com/senamakel/tinybus", rev = "d5b949be8ea56eb5a17289a09aa28e8c09bd88b7", default-features = false } +tinystoragedrivers-testkit = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinystoragedrivers-busproxy = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } +tinybus-sdk = { package = "tinybus", git = "https://github.com/senamakel/tinybus", rev = "0de2d283f35cbbff08efad6fbe05537ffb74611b", default-features = false } From 980b72d39c260aef0243551d1bf4982f70e66bd2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Tue, 27 Oct 2026 04:38:58 +0000 Subject: [PATCH 54/64] test(run-manager): await actor recovery completion Co-authored-by: Medulla --- .../src/run_manager/actor_recovery.rs | 5 ++ .../src/run_manager_actor_recovery_tests.rs | 52 +++++++++++++------ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/crates/tinyflows-module/src/run_manager/actor_recovery.rs b/crates/tinyflows-module/src/run_manager/actor_recovery.rs index 24de6755..c5336a87 100644 --- a/crates/tinyflows-module/src/run_manager/actor_recovery.rs +++ b/crates/tinyflows-module/src/run_manager/actor_recovery.rs @@ -83,9 +83,14 @@ impl Drop for ActorRecoveryGuard { // environment, so even cancellation before its first poll releases // this entry without removing a replacement actor. let active_cleanup = ActiveCleanupGuard(Arc::clone(&self.active)); + #[cfg(test)] + let run_id = self.run_id.clone(); runtime.spawn(async move { let _active_cleanup = active_cleanup; recovery.reconcile().await; + drop(_active_cleanup); + #[cfg(test)] + super::actor_recovery_tests::notify_recovery_finished_for_test(&run_id); }); } } diff --git a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs index b0d0fa04..ccdaf68e 100644 --- a/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs +++ b/crates/tinyflows-module/src/run_manager_actor_recovery_tests.rs @@ -31,7 +31,12 @@ async fn dropping_an_unpolled_actor_future_reconciles_only_its_live_run() { // Tokio can drop a spawned future before its first poll during shutdown. // The actor factory must have captured recovery ownership before it // returned this future, so the admitted row cannot be left running. + let (reconciled_tx, reconciled_rx) = oneshot::channel(); + install_recovery_completion(&fixture.run_id, reconciled_tx); drop(actor); + reconciled_rx + .await + .expect("actor recovery task must finish"); assert_reconciled( &fixture.catalog, @@ -60,9 +65,14 @@ async fn aborting_an_admitted_actor_reconciles_its_running_row() { )); admitted_tx.send(()).unwrap(); arrived_rx.await.unwrap(); + let (reconciled_tx, reconciled_rx) = oneshot::channel(); + install_recovery_completion(&fixture.run_id, reconciled_tx); task.abort(); assert!(task.await.unwrap_err().is_cancelled()); drop(release_tx); + reconciled_rx + .await + .expect("actor recovery task must finish"); assert_reconciled( &fixture.catalog, @@ -371,23 +381,8 @@ async fn assert_reconciled( run_id: &str, storage: &CatalogStorage, ) { - let mut settled = None; - for _ in 0..128 { - let row = catalog.get_flow_run(run_id).await.unwrap().unwrap(); - let events = catalog.flow_run_events_after(run_id, 0, 20).await.unwrap(); - if row.status == "interrupted" - && catalog - .flow_run_terminal_event_seq(run_id) - .await - .unwrap() - .is_some() - { - settled = Some((row, events)); - break; - } - tokio::task::yield_now().await; - } - let (row, events) = settled.expect("actor recovery must finish its durable cleanup"); + let row = catalog.get_flow_run(run_id).await.unwrap().unwrap(); + let events = catalog.flow_run_events_after(run_id, 0, 20).await.unwrap(); assert_eq!(row.status, "interrupted"); assert!( row.error @@ -465,6 +460,29 @@ fn install_after_admission_gate( static AFTER_ADMISSION_GATES: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +#[cfg(test)] +static RECOVERY_COMPLETIONS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +#[cfg(test)] +fn install_recovery_completion(run_id: &str, finished: oneshot::Sender<()>) { + RECOVERY_COMPLETIONS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(run_id.to_owned(), finished); +} + +#[cfg(test)] +pub(super) fn notify_recovery_finished_for_test(run_id: &str) { + if let Some(finished) = RECOVERY_COMPLETIONS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(run_id) + { + let _ = finished.send(()); + } +} + #[cfg(test)] pub(super) async fn pause_after_admission_for_test(run_id: &str) { let gate = AFTER_ADMISSION_GATES From 85c91f704027526792844adae986598bc3a7214a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:54:35 +0000 Subject: [PATCH 55/64] test(schedule): add identity test module for schedule types Register a new identity_tests module under cfg(test) so the schedule types can be covered by dedicated identity tests. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/schedule/identity_tests.rs | 30 +++++++++++++++++++ crates/tinyflows-bus/src/schedule/types.rs | 4 +++ 2 files changed, 34 insertions(+) create mode 100644 crates/tinyflows-bus/src/schedule/identity_tests.rs diff --git a/crates/tinyflows-bus/src/schedule/identity_tests.rs b/crates/tinyflows-bus/src/schedule/identity_tests.rs new file mode 100644 index 00000000..4dfeb8bc --- /dev/null +++ b/crates/tinyflows-bus/src/schedule/identity_tests.rs @@ -0,0 +1,30 @@ +use super::*; +use serde_json::json; + +#[test] +fn channel_job_origin_roundtrips_session_and_conversation_identity() { + let wire = json!({ + "kind": "channel", + "channel": "telegram", + "reply_target": "42", + "history_key": "telegram_alice_42", + "channel_session": { + "tenant_key": "profile:alice", + "agent_id": "orchestrator", + "session_key": "main:telegram:bot-a:group:room-42", + "provider": "telegram", + "account_id": "telegram:bot-a", + "conversation": { + "kind": "group", + "id": "room-42", + "scope_id": "guild-1", + "parent_id": "parent-1", + "thread_id": "thread-1", + "topic_id": "topic-1" + } + } + }); + + let origin: JobOrigin = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(origin).unwrap(), wire); +} diff --git a/crates/tinyflows-bus/src/schedule/types.rs b/crates/tinyflows-bus/src/schedule/types.rs index bbe61623..39421ce7 100644 --- a/crates/tinyflows-bus/src/schedule/types.rs +++ b/crates/tinyflows-bus/src/schedule/types.rs @@ -470,3 +470,7 @@ pub struct CronJobPatch { #[cfg(test)] #[path = "types_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "identity_tests.rs"] +mod identity_tests; From 004f3b7fb16bfb0f99a66b3b340a4c3cd9242fcd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:55:27 +0000 Subject: [PATCH 56/64] feat(schedule): add channel session identity to job origins Channel job origins can now carry a host-derived session identity describing the tenant, agent, provider, and conversation coordinates, with optional fields omitted from the wire format so legacy origins serialize unchanged. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-bus/src/schedule/mod.rs | 5 +- crates/tinyflows-bus/src/schedule/types.rs | 46 +++++++++++++++++++ .../tinyflows-bus/src/schedule/types_tests.rs | 2 + .../tinyflows-bus/src/schedule/wire_tests.rs | 1 + .../src/schedule/test_support.rs | 1 + .../src/validation_tests.rs | 1 + crates/tinyflows-schedule/src/wire_tests.rs | 1 + 7 files changed, 55 insertions(+), 2 deletions(-) diff --git a/crates/tinyflows-bus/src/schedule/mod.rs b/crates/tinyflows-bus/src/schedule/mod.rs index a36c1cac..e1c66ef1 100644 --- a/crates/tinyflows-bus/src/schedule/mod.rs +++ b/crates/tinyflows-bus/src/schedule/mod.rs @@ -21,8 +21,9 @@ pub use store_types::{ AgentJobSpec, ImportedJob, MAX_CRON_OUTPUT_BYTES, StoreOperation, TRUNCATED_OUTPUT_MARKER, }; pub use types::{ - ActiveHours, CronJob, CronJobPatch, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, - JobPolicy, JobType, Schedule, SessionTarget, delivery_mode, + ActiveHours, ChannelConversationIdentity, ChannelSessionIdentity, CronJob, CronJobPatch, + CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobPolicy, JobType, Schedule, + SessionTarget, delivery_mode, }; /// The module has one bus identity and separately served authoring/schedule objects. diff --git a/crates/tinyflows-bus/src/schedule/types.rs b/crates/tinyflows-bus/src/schedule/types.rs index 39421ce7..1d75ca92 100644 --- a/crates/tinyflows-bus/src/schedule/types.rs +++ b/crates/tinyflows-bus/src/schedule/types.rs @@ -98,9 +98,55 @@ pub enum JobOrigin { /// Channel-side thread or topic within the chat. #[serde(default, skip_serializing_if = "Option::is_none")] thread_id: Option, + /// Trusted host-derived identity for the channel session and + /// conversation. Absent on legacy channel origins. + #[serde(default, skip_serializing_if = "Option::is_none")] + channel_session: Option>, }, } +/// Stable, host-derived identity for a channel session. +/// +/// Optional fields are omitted when absent so older serialized origins retain +/// their existing wire representation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChannelSessionIdentity { + /// Tenant or profile owning the session. + pub tenant_key: String, + /// Agent associated with the session. + pub agent_id: String, + /// Stable key for the session in its target. + pub session_key: String, + /// Channel provider, such as `telegram` or `discord`. + pub provider: String, + /// Provider account that owns the session, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub account_id: Option, + /// Conversation coordinates within the session. + pub conversation: ChannelConversationIdentity, +} + +/// Provider-independent identity of a conversation within a channel session. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChannelConversationIdentity { + /// Conversation category, such as `direct`, `group`, or `channel`. + pub kind: String, + /// Provider-local conversation identifier. + pub id: String, + /// Parent scope, such as a guild or workspace, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope_id: Option, + /// Parent conversation identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + /// Provider thread identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread_id: Option, + /// Provider topic identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option, +} + impl JobOrigin { /// The serialized `kind` tag: `"web"` or `"channel"`. pub fn kind_str(&self) -> &'static str { diff --git a/crates/tinyflows-bus/src/schedule/types_tests.rs b/crates/tinyflows-bus/src/schedule/types_tests.rs index f61533f9..fa9c647d 100644 --- a/crates/tinyflows-bus/src/schedule/types_tests.rs +++ b/crates/tinyflows-bus/src/schedule/types_tests.rs @@ -317,6 +317,7 @@ fn job_origin_channel_roundtrips_with_and_without_optionals() { history_key: "telegram:chat-42".into(), sender: Some("alice".into()), thread_id: Some("topic-7".into()), + channel_session: None, }; let v = serde_json::to_value(&full).unwrap(); assert_eq!(v["kind"], "channel"); @@ -340,6 +341,7 @@ fn job_origin_channel_roundtrips_with_and_without_optionals() { history_key: "discord:c1".into(), sender: None, thread_id: None, + channel_session: None, } ); assert_eq!(serde_json::to_value(&parsed).unwrap(), minimal); diff --git a/crates/tinyflows-bus/src/schedule/wire_tests.rs b/crates/tinyflows-bus/src/schedule/wire_tests.rs index 0d5593e0..4cb9400d 100644 --- a/crates/tinyflows-bus/src/schedule/wire_tests.rs +++ b/crates/tinyflows-bus/src/schedule/wire_tests.rs @@ -123,6 +123,7 @@ fn origin_wire_bytes() { history_key: "telegram:42".into(), sender: Some("alice".into()), thread_id: Some("7".into()), + channel_session: None, }, ); } diff --git a/crates/tinyflows-drivers/src/schedule/test_support.rs b/crates/tinyflows-drivers/src/schedule/test_support.rs index 4628a21b..7c72f6c0 100644 --- a/crates/tinyflows-drivers/src/schedule/test_support.rs +++ b/crates/tinyflows-drivers/src/schedule/test_support.rs @@ -39,6 +39,7 @@ pub(super) fn channel_origin() -> JobOrigin { history_key: "telegram:chat-42".into(), sender: Some("alice".into()), thread_id: None, + channel_session: None, } } diff --git a/crates/tinyflows-schedule/src/validation_tests.rs b/crates/tinyflows-schedule/src/validation_tests.rs index 8de72843..619d11ee 100644 --- a/crates/tinyflows-schedule/src/validation_tests.rs +++ b/crates/tinyflows-schedule/src/validation_tests.rs @@ -48,6 +48,7 @@ fn origin_requires_conversation_and_announce_can_reply_to_its_asker() { history_key: "conversation".into(), sender: None, thread_id: None, + channel_session: None, }; assert!(validate_delivery(&delivery, Some(&origin), None).is_ok()); let announce = DeliveryConfig { diff --git a/crates/tinyflows-schedule/src/wire_tests.rs b/crates/tinyflows-schedule/src/wire_tests.rs index 71c0a700..52a8c7c1 100644 --- a/crates/tinyflows-schedule/src/wire_tests.rs +++ b/crates/tinyflows-schedule/src/wire_tests.rs @@ -123,6 +123,7 @@ fn origin_wire_bytes() { history_key: "telegram:42".into(), sender: Some("alice".into()), thread_id: Some("7".into()), + channel_session: None, }, ); } From e94e7bfeb16957fd04cbc4f7ec6ca9b380b9d0c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:56:00 +0000 Subject: [PATCH 57/64] refactor(schedule): move channel identity types into their own module Extracted ChannelSessionIdentity and ChannelConversationIdentity from the schedule types module into a dedicated identity module, re-exporting them from the same place so existing imports keep working. This keeps the shared schedule types focused and gives the channel identity definitions room to grow. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-bus/src/schedule/identity.rs | 43 +++++++++++++++++++ crates/tinyflows-bus/src/schedule/mod.rs | 7 +-- crates/tinyflows-bus/src/schedule/types.rs | 43 +------------------ 3 files changed, 48 insertions(+), 45 deletions(-) create mode 100644 crates/tinyflows-bus/src/schedule/identity.rs diff --git a/crates/tinyflows-bus/src/schedule/identity.rs b/crates/tinyflows-bus/src/schedule/identity.rs new file mode 100644 index 00000000..e5244ffa --- /dev/null +++ b/crates/tinyflows-bus/src/schedule/identity.rs @@ -0,0 +1,43 @@ +use serde::{Deserialize, Serialize}; + +/// Stable, host-derived identity for a channel session. +/// +/// Optional fields are omitted when absent so older serialized origins retain +/// their existing wire representation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChannelSessionIdentity { + /// Tenant or profile owning the session. + pub tenant_key: String, + /// Agent associated with the session. + pub agent_id: String, + /// Stable key for the session in its target. + pub session_key: String, + /// Channel provider, such as `telegram` or `discord`. + pub provider: String, + /// Provider account that owns the session, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub account_id: Option, + /// Conversation coordinates within the session. + pub conversation: ChannelConversationIdentity, +} + +/// Provider-independent identity of a conversation within a channel session. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChannelConversationIdentity { + /// Conversation category, such as `direct`, `group`, or `channel`. + pub kind: String, + /// Provider-local conversation identifier. + pub id: String, + /// Parent scope, such as a guild or workspace, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope_id: Option, + /// Parent conversation identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_id: Option, + /// Provider thread identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread_id: Option, + /// Provider topic identifier, when available. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topic_id: Option, +} diff --git a/crates/tinyflows-bus/src/schedule/mod.rs b/crates/tinyflows-bus/src/schedule/mod.rs index e1c66ef1..9ef3befa 100644 --- a/crates/tinyflows-bus/src/schedule/mod.rs +++ b/crates/tinyflows-bus/src/schedule/mod.rs @@ -3,12 +3,14 @@ use serde::{Deserialize, Serialize}; mod frequency; +mod identity; mod records; mod requests; mod store_types; pub mod types; pub use frequency::{MIN_AGENT_JOB_INTERVAL, TooFrequent, describe_gap}; +pub use identity::{ChannelConversationIdentity, ChannelSessionIdentity}; pub use records::{Attempt, Completion, RunRecord}; pub use requests::{ AcknowledgeManualRun, CheckOriginRequest, CheckPatchRequest, DeliveryProjection, @@ -21,9 +23,8 @@ pub use store_types::{ AgentJobSpec, ImportedJob, MAX_CRON_OUTPUT_BYTES, StoreOperation, TRUNCATED_OUTPUT_MARKER, }; pub use types::{ - ActiveHours, ChannelConversationIdentity, ChannelSessionIdentity, CronJob, CronJobPatch, - CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, JobPolicy, JobType, Schedule, - SessionTarget, delivery_mode, + ActiveHours, CronJob, CronJobPatch, CronRun, DeliveryConfig, DeliveryStatus, JobOrigin, + JobPolicy, JobType, Schedule, SessionTarget, delivery_mode, }; /// The module has one bus identity and separately served authoring/schedule objects. diff --git a/crates/tinyflows-bus/src/schedule/types.rs b/crates/tinyflows-bus/src/schedule/types.rs index 1d75ca92..b7bb562b 100644 --- a/crates/tinyflows-bus/src/schedule/types.rs +++ b/crates/tinyflows-bus/src/schedule/types.rs @@ -1,5 +1,6 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Deserializer, Serialize}; +use super::ChannelSessionIdentity; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "lowercase")] @@ -105,48 +106,6 @@ pub enum JobOrigin { }, } -/// Stable, host-derived identity for a channel session. -/// -/// Optional fields are omitted when absent so older serialized origins retain -/// their existing wire representation. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ChannelSessionIdentity { - /// Tenant or profile owning the session. - pub tenant_key: String, - /// Agent associated with the session. - pub agent_id: String, - /// Stable key for the session in its target. - pub session_key: String, - /// Channel provider, such as `telegram` or `discord`. - pub provider: String, - /// Provider account that owns the session, when available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub account_id: Option, - /// Conversation coordinates within the session. - pub conversation: ChannelConversationIdentity, -} - -/// Provider-independent identity of a conversation within a channel session. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ChannelConversationIdentity { - /// Conversation category, such as `direct`, `group`, or `channel`. - pub kind: String, - /// Provider-local conversation identifier. - pub id: String, - /// Parent scope, such as a guild or workspace, when available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub scope_id: Option, - /// Parent conversation identifier, when available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_id: Option, - /// Provider thread identifier, when available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub thread_id: Option, - /// Provider topic identifier, when available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub topic_id: Option, -} - impl JobOrigin { /// The serialized `kind` tag: `"web"` or `"channel"`. pub fn kind_str(&self) -> &'static str { From 5fbd20bf41b4413ac6fba24613bd1c1c7e6a7e26 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:56:16 +0000 Subject: [PATCH 58/64] style(tinyflows-bus): reorder imports in schedule types Move the local ChannelSessionIdentity import above the external crates so the import grouping follows the project's convention. No behaviour change. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-bus/src/schedule/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyflows-bus/src/schedule/types.rs b/crates/tinyflows-bus/src/schedule/types.rs index b7bb562b..665eaf2f 100644 --- a/crates/tinyflows-bus/src/schedule/types.rs +++ b/crates/tinyflows-bus/src/schedule/types.rs @@ -1,6 +1,6 @@ +use super::ChannelSessionIdentity; use chrono::{DateTime, Utc}; use serde::{Deserialize, Deserializer, Serialize}; -use super::ChannelSessionIdentity; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "lowercase")] From 53266edcd24410a8cf33f06b0ccc519e992cdc95 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:56:54 +0000 Subject: [PATCH 59/64] test(schedule): build channel origin directly in identity roundtrip test The roundtrip test now constructs a JobOrigin::Channel value and serializes it instead of parsing a hand-written JSON literal, then asserts the nested session and conversation fields survive the roundtrip. This exercises the typed identity structures rather than only the deserializer. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/schedule/identity_tests.rs | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/tinyflows-bus/src/schedule/identity_tests.rs b/crates/tinyflows-bus/src/schedule/identity_tests.rs index 4dfeb8bc..c2c233c1 100644 --- a/crates/tinyflows-bus/src/schedule/identity_tests.rs +++ b/crates/tinyflows-bus/src/schedule/identity_tests.rs @@ -1,30 +1,34 @@ use super::*; +use crate::schedule::{ChannelConversationIdentity, ChannelSessionIdentity}; use serde_json::json; #[test] fn channel_job_origin_roundtrips_session_and_conversation_identity() { - let wire = json!({ - "kind": "channel", - "channel": "telegram", - "reply_target": "42", - "history_key": "telegram_alice_42", - "channel_session": { - "tenant_key": "profile:alice", - "agent_id": "orchestrator", - "session_key": "main:telegram:bot-a:group:room-42", - "provider": "telegram", - "account_id": "telegram:bot-a", - "conversation": { - "kind": "group", - "id": "room-42", - "scope_id": "guild-1", - "parent_id": "parent-1", - "thread_id": "thread-1", - "topic_id": "topic-1" - } - } - }); + let origin = JobOrigin::Channel { + channel: "telegram".into(), + reply_target: "42".into(), + history_key: "telegram_alice_42".into(), + sender: Some("alice".into()), + thread_id: None, + channel_session: Some(Box::new(ChannelSessionIdentity { + tenant_key: "profile:alice".into(), + agent_id: "orchestrator".into(), + session_key: "main:telegram:bot-a:group:room-42".into(), + provider: "telegram".into(), + account_id: Some("telegram:bot-a".into()), + conversation: ChannelConversationIdentity { + kind: "group".into(), + id: "room-42".into(), + scope_id: Some("guild-1".into()), + parent_id: Some("parent-1".into()), + thread_id: Some("thread-1".into()), + topic_id: Some("topic-1".into()), + }, + })), + }; - let origin: JobOrigin = serde_json::from_value(wire.clone()).unwrap(); - assert_eq!(serde_json::to_value(origin).unwrap(), wire); + let wire = serde_json::to_value(&origin).unwrap(); + assert_eq!(wire["channel_session"]["tenant_key"], "profile:alice"); + assert_eq!(wire["channel_session"]["conversation"]["thread_id"], "thread-1"); + assert_eq!(serde_json::from_value::(wire).unwrap(), origin); } From 6a455a65da4696adc6d3e57624f9690afa9808b6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:57:06 +0000 Subject: [PATCH 60/64] test(schedule): reformat thread id assertion for line length Wrap the thread_id assertion in the identity roundtrip test across multiple lines so it stays within the formatter's line width. No behaviour change. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-bus/src/schedule/identity_tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tinyflows-bus/src/schedule/identity_tests.rs b/crates/tinyflows-bus/src/schedule/identity_tests.rs index c2c233c1..5a2a5c40 100644 --- a/crates/tinyflows-bus/src/schedule/identity_tests.rs +++ b/crates/tinyflows-bus/src/schedule/identity_tests.rs @@ -29,6 +29,9 @@ fn channel_job_origin_roundtrips_session_and_conversation_identity() { let wire = serde_json::to_value(&origin).unwrap(); assert_eq!(wire["channel_session"]["tenant_key"], "profile:alice"); - assert_eq!(wire["channel_session"]["conversation"]["thread_id"], "thread-1"); + assert_eq!( + wire["channel_session"]["conversation"]["thread_id"], + "thread-1" + ); assert_eq!(serde_json::from_value::(wire).unwrap(), origin); } From f64c31d149935b30e593a569a346659bce5fd78a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 11:57:19 +0000 Subject: [PATCH 61/64] test(schedule): remove unused serde_json import from identity tests Drops an unused `serde_json::json` import from the schedule identity tests, keeping the test module free of warnings. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-bus/src/schedule/identity_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tinyflows-bus/src/schedule/identity_tests.rs b/crates/tinyflows-bus/src/schedule/identity_tests.rs index 5a2a5c40..1e4c904c 100644 --- a/crates/tinyflows-bus/src/schedule/identity_tests.rs +++ b/crates/tinyflows-bus/src/schedule/identity_tests.rs @@ -1,6 +1,5 @@ use super::*; use crate::schedule::{ChannelConversationIdentity, ChannelSessionIdentity}; -use serde_json::json; #[test] fn channel_job_origin_roundtrips_session_and_conversation_identity() { From 17c881b79ea4bcec2af16adbe12ca25cd8b44fa1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 17:43:50 +0000 Subject: [PATCH 62/64] chore(storage): document git pin rationale for tinystoragedrivers Add a comment explaining that the exact git revisions bridge the storage security migration while upstream roll-ups are pending, noting the unreleased Storage Bus/QueueStore API and post-release security fixes they include. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyflows-storage/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/tinyflows-storage/Cargo.toml b/crates/tinyflows-storage/Cargo.toml index a565e2ea..f6378d0c 100644 --- a/crates/tinyflows-storage/Cargo.toml +++ b/crates/tinyflows-storage/Cargo.toml @@ -9,6 +9,11 @@ publish = false description = "TinyFlows-owned generic document persistence helpers" [dependencies] +# These exact git pins bridge the storage security migration while its +# upstream roll-ups are pending. Relative to the published 2.1.2 crates, the +# pin includes the unreleased Storage Bus/QueueStore API (tinystoragedrivers +# PRs #71 and #72) and post-release workspace/lockfile security fixes (#73). +# The consumed API/type crates are otherwise unchanged from 2.1.2. tinystoragedrivers-api = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } tinystoragedrivers-types = { git = "https://github.com/senamakel/tinystoragedrivers", version = "2.1.2", rev = "2d23a6a6009fa52d1e7aa18de589bdd109a54b6a" } serde = { workspace = true } From c099bee408f256fd323a1ca8daf790432bfd9901 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 30 Oct 2026 17:50:19 +0000 Subject: [PATCH 63/64] refactor(leases): move advanced lease operations into a submodule The acquire_with, inspect and release_epoch methods were relocated into a new leases/advanced.rs submodule, wired in via a path attribute, with no change to their behaviour. This keeps the core lease primitives file focused and gives the host-level takeover semantics their own home. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/primitives/leases.rs | 154 +----------------- .../src/primitives/leases/advanced.rs | 114 +++++++++++++ 2 files changed, 117 insertions(+), 151 deletions(-) create mode 100644 crates/tinyflows-storage/src/primitives/leases/advanced.rs diff --git a/crates/tinyflows-storage/src/primitives/leases.rs b/crates/tinyflows-storage/src/primitives/leases.rs index 71bbaa15..f40e1d13 100644 --- a/crates/tinyflows-storage/src/primitives/leases.rs +++ b/crates/tinyflows-storage/src/primitives/leases.rs @@ -22,6 +22,9 @@ use crate::{Clock, system_clock}; use tinystoragedrivers_api::DocumentStore; use tinystoragedrivers_types::{Result, StorageError}; +#[path = "leases/advanced.rs"] +mod advanced; + /// The stored body: `{holder, epoch, expires_ms}` plus the optional /// `endpoint` and `released` fields, both absent from records that never /// used them. @@ -343,157 +346,6 @@ impl Leases { .filter(|found| found.doc.expires_ms > now && !found.doc.released) .map(|found| Lease::from_body(id, found.doc))) } - - /// Take `(owner, key)` with the full host semantics: an endpoint, a - /// distinction between a clean and an unclean takeover, and an unclean - /// takeover of the same holder's leftover live record. - /// - /// The rule, in order: - /// - /// 1. no record: epoch 1, clean; - /// 2. live, same holder and `held_epoch == Some(record.epoch)`: - /// re-entrant, the epoch stays, clean; - /// 3. released: epoch + 1, clean; - /// 4. live, same holder, `held_epoch == None` (the caller holds nothing: - /// the previous incarnation died before it expired): epoch + 1, - /// unclean, taken over at once, fencing the old incarnation out. A - /// caller that passes a `held_epoch` which no longer matches is a - /// stale actor and gets [`Acquisition::Held`] instead, and so does - /// a call that lost a race to another acquire of the same holder; - /// 5. expired (`now >= expires_ms`): epoch + 1, unclean; - /// 6. otherwise [`Acquisition::Held`] with the live record. - /// - /// Racing acquirers get exactly one winner. - /// - /// # Errors - /// - /// `InvalidInput` for a non-positive ttl, a storage error, or a backend - /// error when the epoch cannot be advanced. - pub async fn acquire_with(&self, request: Acquire<'_>) -> Result { - let Acquire { - owner, - key, - holder, - ttl_ms, - endpoint, - held_epoch, - } = request; - let id = id(owner, key)?; - let mut unclean = false; - let mut seen: Option = None; - // `Repo::update` calls the closure again only after losing the swap, - // so any call after the first means another writer got in: judge the - // record as that competitor's write, never as a dead incarnation. - let mut attempts = 0_u32; - let stored = self - .repo - .update(&id, |current| { - let (now, expires_ms) = self.deadline(ttl_ms)?; - unclean = false; - seen = None; - let raced = attempts > 0; - attempts += 1; - let (epoch, was_unclean) = match ¤t { - None => (1, false), - Some(found) => { - let live = found.expires_ms > now && !found.released; - if live && found.holder == holder && held_epoch == Some(found.epoch) { - (found.epoch, false) - } else if live && (found.holder != holder || raced || held_epoch.is_some()) - { - // Another holder, a competing call of the same - // holder that just won, or a caller proving it - // holds a stale epoch: the live record stands. - seen.clone_from(¤t); - return Ok(None); - } else { - let next = found - .epoch - .checked_add(1) - .ok_or_else(|| StorageError::backend("lease epoch overflow"))?; - (next, !found.released) - } - } - }; - unclean = was_unclean; - Ok(Some(Body { - holder: holder.to_owned(), - epoch, - expires_ms, - endpoint: endpoint.map(str::to_owned), - released: false, - })) - }) - .await?; - Ok(match stored { - Some((body, _)) => Acquisition::Granted { - lease: Lease::from_body(id, body), - previous_unclean: unclean, - }, - None => match seen { - Some(body) => Acquisition::Held(LeaseRecord::from_body(body)), - None => { - return Err(StorageError::backend( - "lease acquire declined without a live record", - )); - } - }, - }) - } - - /// The stored record of `(owner, key)`, live, lapsed or released. Unlike - /// [`current`](Self::current) it does not hide a record that stopped - /// excluding others, so an observer can tell a clean release from an - /// abandoned lease and read the epoch a takeover would continue. - /// - /// # Errors - /// - /// A storage error. - pub async fn inspect(&self, owner: &str, key: &str) -> Result> { - let id = id(owner, key)?; - Ok(self - .repo - .get(&id) - .await? - .map(|found| LeaseRecord::from_body(found.doc))) - } - - /// Drop the record of `(owner, key)` if and only if it still belongs to - /// the tenure `(holder, epoch)`, live or not. Returns whether a record was - /// removed; a record at another holder or epoch (a takeover wrote it) is - /// left alone. - /// - /// This is the compaction primitive for lease keys that are never - /// acquired again (one lease per cron occurrence, per run, per claim), - /// which would otherwise pile up: [`release`](Self::release) deliberately - /// keeps the record so a takeover continues the epoch, and nothing else - /// ever removes it. - /// - /// # Why dropping is safe - /// - /// tsd keeps expired and released leases for one reason: a takeover reads - /// the old record and writes `epoch + 1`, so a stale holder's fence at the - /// old epoch stays dead. The delete is conditional on holder and epoch, so - /// it can only remove the very tenure the caller knows: any takeover bumps - /// the epoch, and a record that moved on is not touched. A holder's fence token on a - /// removed record refuses every write (a missing fence document is a - /// failed check), so the tenure that was dropped is fenced out for good. - /// - /// What the delete gives up is the epoch history: a later acquire of the - /// same key would restart at epoch 1. The caller therefore must only - /// compact a key that nothing will acquire again (for a cron occurrence: - /// once the job's next run has moved past it), or one whose protected - /// resources no longer hold epoch-1 writers. - /// - /// # Errors - /// - /// A storage error, or a conflict past the retry budget. - pub async fn compact(&self, owner: &str, key: &str, holder: &str, epoch: u64) -> Result { - let id = id(owner, key)?; - self.repo - .delete_if(&id, |lease| lease.holder == holder && lease.epoch == epoch) - .await - } } fn id(owner: &str, key: &str) -> Result { diff --git a/crates/tinyflows-storage/src/primitives/leases/advanced.rs b/crates/tinyflows-storage/src/primitives/leases/advanced.rs new file mode 100644 index 00000000..6694af32 --- /dev/null +++ b/crates/tinyflows-storage/src/primitives/leases/advanced.rs @@ -0,0 +1,114 @@ +use super::{Acquire, Acquisition, Body, Lease, LeaseRecord, Leases, id}; +use tinystoragedrivers_types::{Result, StorageError}; + +impl Leases { + /// Take `(owner, key)` with endpoint and takeover semantics. + /// + /// A missing record starts at epoch 1. A matching live tenure is + /// re-entrant; a released or expired record advances the epoch. A live + /// leftover from the same holder is taken over only when `held_epoch` is + /// absent. A stale epoch or a competing live holder returns `Held`. + /// + /// # Errors + /// + /// `InvalidInput` for a non-positive ttl, a storage error, or a backend + /// error when the epoch cannot be advanced. + pub async fn acquire_with(&self, request: Acquire<'_>) -> Result { + let Acquire { + owner, + key, + holder, + ttl_ms, + endpoint, + held_epoch, + } = request; + let id = id(owner, key)?; + let mut unclean = false; + let mut seen: Option = None; + let mut attempts = 0_u32; + let stored = self + .repo + .update(&id, |current| { + let (now, expires_ms) = self.deadline(ttl_ms)?; + unclean = false; + seen = None; + let raced = attempts > 0; + attempts += 1; + let (epoch, was_unclean) = match ¤t { + None => (1, false), + Some(found) => { + let live = found.expires_ms > now && !found.released; + if live && found.holder == holder && held_epoch == Some(found.epoch) { + (found.epoch, false) + } else if live && (found.holder != holder || raced || held_epoch.is_some()) + { + seen.clone_from(¤t); + return Ok(None); + } else { + let next = found + .epoch + .checked_add(1) + .ok_or_else(|| StorageError::backend("lease epoch overflow"))?; + (next, !found.released) + } + } + }; + unclean = was_unclean; + Ok(Some(Body { + holder: holder.to_owned(), + epoch, + expires_ms, + endpoint: endpoint.map(str::to_owned), + released: false, + })) + }) + .await?; + Ok(match stored { + Some((body, _)) => Acquisition::Granted { + lease: Lease::from_body(id, body), + previous_unclean: unclean, + }, + None => match seen { + Some(body) => Acquisition::Held(LeaseRecord::from_body(body)), + None => { + return Err(StorageError::backend( + "lease acquire declined without a live record", + )); + } + }, + }) + } + + /// Return the stored record, including lapsed and released leases. + /// + /// Unlike [`Self::current`], this lets observers distinguish a clean + /// release from an abandoned lease and read the epoch a takeover resumes. + /// + /// # Errors + /// + /// A storage error. + pub async fn inspect(&self, owner: &str, key: &str) -> Result> { + let id = id(owner, key)?; + Ok(self + .repo + .get(&id) + .await? + .map(|found| LeaseRecord::from_body(found.doc))) + } + + /// Delete a lease only while it still belongs to `(holder, epoch)`. + /// + /// This compacts keys that will never be acquired again. A later acquire + /// would restart at epoch 1, so callers must ensure no protected resource + /// can still accept writes from that earlier epoch. + /// + /// # Errors + /// + /// A storage error, or a conflict past the retry budget. + pub async fn compact(&self, owner: &str, key: &str, holder: &str, epoch: u64) -> Result { + let id = id(owner, key)?; + self.repo + .delete_if(&id, |lease| lease.holder == holder && lease.epoch == epoch) + .await + } +} From 1b9b9b0f9ed76562e7fe919de7381236353c01be Mon Sep 17 00:00:00 2001 From: M3gA-Mind Date: Mon, 2 Nov 2026 23:12:54 +0530 Subject: [PATCH 64/64] fix(schedule): log a background run that did not succeed dispatch_claims dropped the ManualRunOutcome, so a run whose delivery failed returned Ok and reap() never reported it. The run record already persists the result; this adds the missing log line. --- crates/tinyflows-schedule/src/runtime/mod.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/tinyflows-schedule/src/runtime/mod.rs b/crates/tinyflows-schedule/src/runtime/mod.rs index 614ee958..95c1fe04 100644 --- a/crates/tinyflows-schedule/src/runtime/mod.rs +++ b/crates/tinyflows-schedule/src/runtime/mod.rs @@ -198,8 +198,9 @@ impl Scheduler { let config = self.config.clone(); let cancellation = self.cancellation.subscribe(); let permits = Arc::clone(&self.permits); + let job_id = claim.job.id.clone(); self.tasks.spawn(async move { - run_claim_with_renewal( + let outcome = run_claim_with_renewal( &ports, &config, claim, @@ -208,8 +209,17 @@ impl Scheduler { Some(permits), cancellation, ) - .await - .map(|_| ()) + .await?; + // The run record holds the result; a failed delivery has no + // other trace in the poll loop. + if !outcome.success { + tracing::warn!( + %job_id, + delivery_status = ?outcome.delivery_status, + "scheduled run did not succeed" + ); + } + Ok(()) }); count += 1; }