From bab8b5b6e956be55adf2096e9ec6e2d1ca4c8fec Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 01:24:13 +0800 Subject: [PATCH 01/14] perf(mst2): reuse native subtree projections across commits --- src/ceres/api_service/mod.rs | 7 + src/ceres/api_service/mono_api_service.rs | 4 + src/ceres/snapshot/native_projection_tests.rs | 491 ++++++++++++++++++ src/ceres/snapshot/pages.rs | 379 +++++++++++--- src/jupiter/storage/mod.rs | 5 + src/jupiter/tests.rs | 1 + 6 files changed, 813 insertions(+), 74 deletions(-) create mode 100644 src/ceres/snapshot/native_projection_tests.rs diff --git a/src/ceres/api_service/mod.rs b/src/ceres/api_service/mod.rs index 856087b2..e5e0f4be 100644 --- a/src/ceres/api_service/mod.rs +++ b/src/ceres/api_service/mod.rs @@ -60,6 +60,13 @@ mod un25_freeze; pub trait ApiHandler: Send + Sync { fn get_context(&self) -> Storage; + /// Only native monorepo trees use the native snapshot memoization domain. + /// Import and custom handlers remain uncached unless they implement their + /// own source identity and verification contract. + fn native_snapshot_projection(&self) -> bool { + false + } + fn object_cache(&self) -> &GitObjectCache; fn strip_relative(&self, path: &Path) -> Result; diff --git a/src/ceres/api_service/mono_api_service.rs b/src/ceres/api_service/mono_api_service.rs index f499b159..0a204c16 100644 --- a/src/ceres/api_service/mono_api_service.rs +++ b/src/ceres/api_service/mono_api_service.rs @@ -1177,6 +1177,10 @@ impl ApiHandler for MonoApiService { self.storage.clone() } + fn native_snapshot_projection(&self) -> bool { + true + } + fn object_cache(&self) -> &GitObjectCache { &self.git_object_cache } diff --git a/src/ceres/snapshot/native_projection_tests.rs b/src/ceres/snapshot/native_projection_tests.rs new file mode 100644 index 00000000..6fe8f406 --- /dev/null +++ b/src/ceres/snapshot/native_projection_tests.rs @@ -0,0 +1,491 @@ +//! Real persisted commits and object storage, with an independent raw-content +//! oracle. This exercises on-demand memoization, not durable publication. + +use std::{collections::BTreeMap, sync::Arc}; + +use git_internal::{ + hash::{HashKind, ObjectHash}, + internal::object::{ + blob::Blob, + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + types::ObjectType, + }, +}; +use sha2::{Digest, Sha256}; + +use super::{FsKind, ProjectionWork, build_directory_page, build_directory_page_with_work}; +use crate::{ + ceres::api_service::{ApiHandler, cache::GitObjectCache, mono_api_service::MonoApiService}, + config::testing::isolated_config, + jupiter::{ + service::git_service::GitService, + storage::object_storage::build_object_storage, + tests::{test_redis_manager, test_storage_with_config}, + }, +}; + +type Files = BTreeMap>; + +async fn handler(temp: &std::path::Path) -> MonoApiService { + let config = isolated_config(temp.join("config")); + let object_storage = build_object_storage(&config.object_storage).await.unwrap(); + let mut storage = test_storage_with_config(temp, config).await; + storage.git_service = GitService { + obj_storage: object_storage, + }; + MonoApiService { + storage, + git_object_cache: Arc::new(GitObjectCache { + connection: test_redis_manager().await, + prefix: String::new(), + }), + } +} + +#[derive(Default)] +struct Directory { + children: BTreeMap, + files: BTreeMap, +} + +fn build_trees(directory: Directory, trees: &mut Vec) -> Tree { + let mut items: BTreeMap = directory + .files + .into_iter() + .map(|(name, oid)| { + let item = TreeItem::new(TreeItemMode::Blob, oid, name.clone()); + (name, item) + }) + .collect(); + for (name, child) in directory.children { + let child = build_trees(child, trees); + items.insert( + name.clone(), + TreeItem::new(TreeItemMode::Tree, child.id, name), + ); + } + let tree = + Tree::from_tree_items_with_kind(HashKind::Sha1, items.into_values().collect()).unwrap(); + trees.push(tree.clone()); + tree +} + +async fn persist_commit( + handler: &MonoApiService, + files: &Files, + parents: Vec, + message: &str, +) -> (Commit, Tree) { + let mut directory = Directory::default(); + for (path, raw) in files { + let blob = Blob::from_content_bytes_with_kind(HashKind::Sha1, raw.clone()).unwrap(); + handler + .storage + .git_service + .save_object_from_model(raw.clone(), &blob.id.to_string()) + .await + .unwrap(); + let components: Vec<_> = path.trim_start_matches('/').split('/').collect(); + let mut parent = &mut directory; + for component in &components[..components.len() - 1] { + parent = parent.children.entry((*component).to_string()).or_default(); + } + parent + .files + .insert(components.last().unwrap().to_string(), blob.id); + } + let mut trees = Vec::new(); + let root = build_trees(directory, &mut trees); + persist_tree_commit(handler, root, trees, parents, message).await +} + +async fn persist_tree_commit( + handler: &MonoApiService, + root: Tree, + trees: Vec, + parents: Vec, + message: &str, +) -> (Commit, Tree) { + let commit = Commit::from_tree_id_with_kind(HashKind::Sha1, root.id, parents, message).unwrap(); + let mono = handler.storage.mono_storage(); + mono.save_mega_trees(trees, commit.id, None).await.unwrap(); + mono.save_mega_commits(vec![commit.clone()], None) + .await + .unwrap(); + // Read back the real persisted commit and its fixed tree; no fake blob OID + // is used as a commit identity and no live ref is required by the builder. + let persisted = handler + .get_commit_by_hash(&commit.id.to_string()) + .await + .unwrap(); + assert_eq!(persisted.tree_id, root.id); + let root = handler + .get_tree_by_hash(&persisted.tree_id.to_string()) + .await + .unwrap(); + (persisted, root) +} + +fn fixture(modules: usize, buckets: usize, files_per_bucket: usize, bytes: usize) -> Files { + let mut files = Files::new(); + for module in 0..modules { + for bucket in 0..buckets { + for file in 0..files_per_bucket { + let path = format!("/project/m{module:03}/d{bucket:02}/f{file:03}"); + let mut raw = vec![b'x'; bytes.max(path.len())]; + raw[..path.len()].copy_from_slice(path.as_bytes()); + files.insert(path, raw); + } + } + } + files +} + +async fn assert_raw_oracle(handler: &MonoApiService, root: &Tree, scope: &str, files: &Files) { + let mut paths = vec![scope.to_string()]; + let mut projected = BTreeMap::new(); + while let Some(path) = paths.pop() { + let directory = build_directory_page(handler, root, &path).await.unwrap(); + for entry in &directory.entries { + let child = format!("{}/{name}", path.trim_end_matches('/'), name = entry.name); + if entry.fs_kind == FsKind::Directory { + paths.push(child); + } else { + assert_eq!(entry.fs_kind, FsKind::Regular); + projected.insert(child, (entry.size.unwrap(), entry.content_digest.unwrap())); + } + } + } + let expected: BTreeMap<_, _> = files + .iter() + .map(|(path, raw)| { + ( + path.clone(), + (raw.len() as u64, <[u8; 32]>::from(Sha256::digest(raw))), + ) + }) + .collect(); + assert_eq!(projected, expected); +} + +async fn leaf_update_case( + modules: usize, + buckets: usize, + files_per_bucket: usize, + bytes: usize, +) -> ProjectionWork { + let temp = tempfile::tempdir().unwrap(); + let handler = handler(temp.path()).await; + let mut files = fixture(modules, buckets, files_per_bucket, bytes); + let original = files.clone(); + let (v1, root1) = persist_commit(&handler, &files, vec![], "native projection V1").await; + let (page1, cold) = build_directory_page_with_work(&handler, &root1, "/project") + .await + .unwrap(); + assert_eq!( + cold.directories_rebuilt, + (1 + modules + modules * buckets) as u64 + ); + assert_eq!(cold.reused_subtree_roots, 0); + assert_eq!(cold.directory_root_pages_built, cold.directories_rebuilt); + assert_eq!(cold.verified_blob_misses, files.len() as u64); + + files.get_mut("/project/m000/d00/f000").unwrap().push(b'2'); + let (v2, root2) = persist_commit(&handler, &files, vec![v1.id], "native projection V2").await; + let (page2, update) = build_directory_page_with_work(&handler, &root2, "/project") + .await + .unwrap(); + assert_ne!(page1.page_id, page2.page_id); + assert_eq!(update.directories_rebuilt, 3); + assert_eq!(update.directory_root_pages_built, 3); + assert_eq!( + update.tree_fetches, 3, + "scope + two changed child trees only" + ); + assert_eq!( + update.reused_subtree_roots, + (modules - 1 + buckets - 1) as u64 + ); + assert_eq!( + update.directory_entries_scanned, + (modules + buckets + files_per_bucket) as u64 + ); + assert_eq!(update.verified_blob_hits, (files_per_bucket - 1) as u64); + assert_eq!(update.verified_blob_misses, 1); + assert_eq!( + update.raw_bytes_fetched, + files["/project/m000/d00/f000"].len() as u64 + ); + assert_eq!(update.raw_bytes_hashed, update.raw_bytes_fetched); + let scope_tree = + super::fetch_tree_with_work(&handler, &root2, "/project", &mut ProjectionWork::default()) + .await + .unwrap(); + let mut full_work = ProjectionWork::default(); + let full = super::build_subtree( + &handler, + &handler.storage, + &scope_tree, + "/project", + false, + &mut full_work, + ) + .await + .unwrap(); + assert_eq!( + page2.page_id, full.page_id, + "cache reuse equals a fresh full projection" + ); + assert_eq!(full_work.directories_rebuilt, cold.directories_rebuilt); + assert_eq!(full_work.reused_subtree_roots, 0); + assert_raw_oracle(&handler, &root2, "/project", &files).await; + + // A third real commit repeats the operation; this cannot be a special + // first-update shortcut. Reading older fixed roots remains correct. + let second_files = files.clone(); + files.get_mut("/project/m000/d00/f000").unwrap().push(b'3'); + let (_, root3) = persist_commit(&handler, &files, vec![v2.id], "native projection V3").await; + let (_, third) = build_directory_page_with_work(&handler, &root3, "/project") + .await + .unwrap(); + assert_eq!(third.directories_rebuilt, update.directories_rebuilt); + assert_eq!( + third.directory_entries_scanned, + update.directory_entries_scanned + ); + assert_eq!(third.reused_subtree_roots, update.reused_subtree_roots); + assert_eq!(third.verified_blob_misses, 1); + assert_raw_oracle(&handler, &root3, "/project", &files).await; + assert_raw_oracle(&handler, &root2, "/project", &second_files).await; + + let (old, old_work) = build_directory_page_with_work(&handler, &root1, "/project") + .await + .unwrap(); + assert_eq!(old.page_id, page1.page_id); + assert_eq!(old_work.directories_rebuilt, 0); + assert_eq!(old_work.directory_entries_scanned, 0); + assert_eq!(old_work.reused_subtree_roots, 1); + assert_raw_oracle(&handler, &root1, "/project", &original).await; + let old_file = super::resolve_abs(&handler, &root1, "/project/m000/d00/f000") + .await + .unwrap(); + assert!( + matches!(old_file, super::WalkOutcome::FoundFile { raw, .. } if raw == original["/project/m000/d00/f000"]) + ); + update +} + +#[tokio::test] +async fn mst2_native_projection_real_commit_leaf_update_only_rebuilds_ancestors() { + leaf_update_case(4, 3, 4, 64).await; +} + +#[tokio::test] +#[ignore = "opt-in 16,384-file/256MiB real object-storage projection workload"] +async fn mst2_native_projection_medium_real_commit_leaf_update() { + let update = leaf_update_case(64, 8, 32, 16 * 1024).await; + assert_eq!(update.directory_entries_scanned, 104); + assert_eq!(update.reused_subtree_roots, 70); + eprintln!("native projection memoization work: {update:?}"); +} + +#[tokio::test] +async fn mst2_native_projection_same_oid_isolated_storage_does_not_share_cache() { + let temp_a = tempfile::tempdir().unwrap(); + let temp_b = tempfile::tempdir().unwrap(); + let a = handler(temp_a.path()).await; + let b = handler(temp_b.path()).await; + let files = fixture(2, 2, 2, 64); + let (_, root) = persist_commit(&a, &files, vec![], "independent storage A").await; + let (cached, _) = build_directory_page_with_work(&a, &root, "/") + .await + .unwrap(); + assert!( + build_directory_page_with_work(&b, &root, "/") + .await + .is_err(), + "B must fetch its own missing child tree" + ); + let (_, own_root) = persist_commit(&b, &files, vec![], "independent storage B").await; + assert_eq!(root.id, own_root.id); + let (rebuilt, work) = build_directory_page_with_work(&b, &own_root, "/") + .await + .unwrap(); + assert_eq!(cached.page_id, rebuilt.page_id); + assert_eq!(work.directories_rebuilt, 8); + assert_eq!(work.reused_subtree_roots, 0); + let clone = a.clone(); + let (_, clone_work) = build_directory_page_with_work(&clone, &root, "/") + .await + .unwrap(); + assert_eq!(clone_work.directories_rebuilt, 0); + assert_eq!(clone_work.tree_fetches, 0); +} + +#[tokio::test] +async fn mst2_native_projection_real_commit_directory_move_reuses_identical_tree() { + let temp = tempfile::tempdir().unwrap(); + let handler = handler(temp.path()).await; + let files = Files::from([("/project/old/nested/file".into(), b"move me".to_vec())]); + let (v1, root1) = persist_commit(&handler, &files, vec![], "before move").await; + let (before, _) = build_directory_page_with_work(&handler, &root1, "/project") + .await + .unwrap(); + let moved = Files::from([("/project/new/nested/file".into(), b"move me".to_vec())]); + let (_, root2) = persist_commit(&handler, &moved, vec![v1.id], "after move").await; + let (after, work) = build_directory_page_with_work(&handler, &root2, "/project") + .await + .unwrap(); + assert_eq!( + before.entries[0].directory_root, + after.entries[0].directory_root + ); + assert_eq!(work.directories_rebuilt, 1); + assert_eq!(work.reused_subtree_roots, 1); + assert_eq!( + work.tree_fetches, 1, + "only scope; moved root is reused by OID" + ); + assert_eq!(work.verified_blob_hits + work.verified_blob_misses, 0); + assert_raw_oracle(&handler, &root2, "/project", &moved).await; +} + +fn byte_prefix(bytes: usize) -> String { + let mut path = "/project".to_string(); + while path.len() < bytes { + let length = (bytes - path.len() - 1).min(255); + assert_ne!(length, 0); + path.push('/'); + path.extend(std::iter::repeat_n('a', length)); + } + path +} + +fn wrap_tree_at(prefix: &str, mut child: Tree, trees: &mut Vec) -> Tree { + for name in prefix[1..].split('/').rev() { + child = Tree::from_tree_items_with_kind( + HashKind::Sha1, + vec![TreeItem::new( + TreeItemMode::Tree, + child.id, + name.to_string(), + )], + ) + .unwrap(); + trees.push(child.clone()); + } + child +} + +#[tokio::test] +async fn mst2_native_projection_cached_empty_child_move_rechecks_path_budgets() { + let temp = tempfile::tempdir().unwrap(); + let handler = handler(temp.path()).await; + let empty = Tree { + id: ObjectHash::from_type_and_data_for_kind(HashKind::Sha1, ObjectType::Tree, &[]).unwrap(), + tree_items: vec![], + }; + let source = Tree::from_tree_items_with_kind( + HashKind::Sha1, + vec![TreeItem::new( + TreeItemMode::Tree, + empty.id, + "empty".to_string(), + )], + ) + .unwrap(); + let mut trees = vec![empty.clone(), source.clone()]; + let root = wrap_tree_at("/project/source", source.clone(), &mut trees); + let (mut commit, root) = + persist_tree_commit(&handler, root, trees, vec![], "empty source").await; + build_directory_page_with_work(&handler, &root, "/project/source") + .await + .unwrap(); + for (index, (prefix, allowed)) in [ + (byte_prefix(4090), true), + (byte_prefix(4091), false), + (format!("/project/{}", vec!["a"; 254].join("/")), true), + (format!("/project/{}", vec!["a"; 255].join("/")), false), + ] + .into_iter() + .enumerate() + { + let mut trees = vec![empty.clone(), source.clone()]; + let root = wrap_tree_at(&prefix, source.clone(), &mut trees); + let (next, root) = persist_tree_commit( + &handler, + root, + trees, + vec![commit.id], + &format!("empty move {index}"), + ) + .await; + commit = next; + let result = build_directory_page_with_work(&handler, &root, &prefix).await; + if allowed { + let (directory, work) = result.unwrap(); + assert_eq!(directory.entries[0].name, "empty"); + assert_eq!(work.directories_rebuilt, 0); + assert_eq!(work.reused_subtree_roots, 1); + } else { + assert!( + matches!(result, Err(error) if error.code == super::SnapshotErrorCode::ScopeInvalid) + ); + } + } +} + +#[test] +fn mst2_native_projection_cache_separates_equal_hex_different_hash_kinds() { + let cache = super::NativeProjectionCache::default(); + let sha = ObjectHash::from_bytes_for_kind(HashKind::Sha256, &[7; 32]).unwrap(); + let blake = ObjectHash::from_bytes_for_kind(HashKind::Blake3, &[7; 32]).unwrap(); + assert_eq!(sha.to_string(), blake.to_string()); + let page_bytes = mst2_codec::metapage::Page::build(&[]).unwrap(); + let directory = Arc::new(super::BuiltDirectory { + page_id: mst2_codec::metapage::page_id(&page_bytes), + page_bytes, + entries: vec![], + codec_entries: vec![], + path_budget: super::DescendantPathBudget::default(), + }); + cache.insert(sha, Arc::clone(&directory)); + assert!(cache.get(blake).is_none()); + assert!(Arc::ptr_eq(&cache.get(sha).unwrap(), &directory)); +} + +#[tokio::test] +async fn mst2_native_projection_cached_move_rechecks_full_scope_path_budgets() { + let temp = tempfile::tempdir().unwrap(); + let handler = handler(temp.path()).await; + let raw = b"multibyte basename".to_vec(); + let files = Files::from([("/project/source/é".into(), raw.clone())]); + let (mut commit, root) = persist_commit(&handler, &files, vec![], "warm source").await; + build_directory_page_with_work(&handler, &root, "/project/source") + .await + .unwrap(); + let prefixes = [ + (byte_prefix(4093), true), + (byte_prefix(4094), false), + (format!("/project/{}", vec!["a"; 254].join("/")), true), + (format!("/project/{}", vec!["a"; 255].join("/")), false), + ]; + for (index, (prefix, allowed)) in prefixes.into_iter().enumerate() { + let files = Files::from([(format!("{prefix}/é"), raw.clone())]); + let (next, root) = + persist_commit(&handler, &files, vec![commit.id], &format!("move {index}")).await; + commit = next; + let result = build_directory_page_with_work(&handler, &root, &prefix).await; + if allowed { + let (_, work) = result.unwrap(); + assert_eq!(work.directories_rebuilt, 0); + assert_eq!(work.reused_subtree_roots, 1); + assert_eq!(work.directory_entries_scanned, 0); + } else { + assert!( + matches!(result, Err(error) if error.code == super::SnapshotErrorCode::ScopeInvalid) + ); + } + } +} diff --git a/src/ceres/snapshot/pages.rs b/src/ceres/snapshot/pages.rs index 555ed913..f19e15fe 100644 --- a/src/ceres/snapshot/pages.rs +++ b/src/ceres/snapshot/pages.rs @@ -6,12 +6,20 @@ //! yields identical page_ids across requests. use std::{ + borrow::Cow, collections::HashMap, - sync::{Arc, Mutex, OnceLock}, + sync::{Arc, Mutex}, }; use base64::Engine; -use mst2_codec::metapage::{Entry, EntryKind, Page, page_id}; +use git_internal::{hash::ObjectHash, internal::object::tree::Tree}; +use mst2_codec::{ + descriptor::{ + ACCESS_PROJECTION_EXACT_FULL, FS_SEMANTICS_LINUX_CODE_V1, + MATERIALIZATION_POLICY_GIT_RAW_V1, METADATA_CODEC, SCHEMA_VERSION, + }, + metapage::{Entry, EntryKind, Page, page_id}, +}; use sea_orm::ActiveValue::Set; use sha2::{Digest, Sha256}; @@ -24,7 +32,7 @@ use crate::{ view::hex, }, }, - jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION, + jupiter::storage::{Storage, mono_storage::MST2_VERIFICATION_VERSION}, }; /// A directory page plus everything the JSON layer needs. @@ -38,6 +46,153 @@ pub struct BuiltDirectory { /// The codec entries the page was built from, so callers can walk a route /// through the same canonical tree (`Page::pages_along_route`). pub codec_entries: Vec, + /// Bounds proven while validating this subtree; rechecked at every new + /// prefix so an identical tree cannot bypass full-path limits after a move. + path_budget: DescendantPathBudget, +} + +#[derive(Debug, Clone, Copy, Default)] +struct DescendantPathBudget { + /// Includes the slash before each descendant name. + suffix_bytes: usize, + components: usize, +} + +impl DescendantPathBudget { + fn include(&mut self, name: &str, child: Self) { + self.suffix_bytes = self.suffix_bytes.max(1 + name.len() + child.suffix_bytes); + self.components = self.components.max(1 + child.components); + } + + fn validate_at(self, prefix: &str) -> Result<(), SnapshotError> { + crate::ceres::snapshot::view::validate_scope_relative_path(prefix)?; + let (bytes, components) = if prefix == "/" { + (0, 0) + } else { + (prefix.len(), prefix[1..].split('/').count()) + }; + if bytes + self.suffix_bytes > 4096 || components + self.components > 256 { + return Err(SnapshotError::new( + SnapshotErrorCode::ScopeInvalid, + "subtree exceeds full-path budget at this prefix", + )); + } + Ok(()) + } +} + +/// Actual work in one on-demand native projection. These are memoization +/// counters, not durable publication or retention accounting. Page counters +/// cover returned directory-root pages only: the codec does not expose the +/// radix pages it internally encodes while building a root. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProjectionWork { + pub directories_rebuilt: u64, + pub reused_subtree_roots: u64, + pub directory_root_pages_built: u64, + pub directory_root_page_bytes_built: u64, + pub directory_root_pages_reused: u64, + pub directory_root_page_bytes_reused: u64, + pub directory_entries_scanned: u64, + pub scope_path_entries_examined: u64, + /// Backend tree requests, excluding the caller-supplied fixed root tree. + pub tree_fetches: u64, + pub verified_blob_hits: u64, + pub verified_blob_misses: u64, + pub raw_bytes_fetched: u64, + pub raw_bytes_hashed: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct NativeProjectionKey { + source_domain: &'static str, + /// Tagged OID preserves SHA-256 vs BLAKE3 even though both are 64 hex. + tree_oid: String, + schema_version: u16, + metadata_codec: u16, + materialization_policy: u16, + fs_semantics: u16, + access_projection: u16, + verification_revision: i32, + projection_revision: u16, +} + +impl NativeProjectionKey { + fn new(oid: ObjectHash) -> Self { + Self { + source_domain: "native-git", + tree_oid: oid.to_tagged_string(), + schema_version: SCHEMA_VERSION, + metadata_codec: METADATA_CODEC, + materialization_policy: MATERIALIZATION_POLICY_GIT_RAW_V1, + fs_semantics: FS_SEMANTICS_LINUX_CODE_V1, + access_projection: ACCESS_PROJECTION_EXACT_FULL, + verification_revision: MST2_VERIFICATION_VERSION, + projection_revision: 1, + } + } +} + +/// Storage-owned, disposable derived facts. This cache grants no authorization +/// and provides no durable retention evidence. Clearing it only costs a rebuild. +#[derive(Default)] +pub(crate) struct NativeProjectionCache { + state: Mutex, +} + +#[derive(Default)] +struct NativeProjectionCacheState { + pages: HashMap>, + retained_payload_bytes: usize, +} + +impl NativeProjectionCache { + fn get(&self, oid: ObjectHash) -> Option> { + // A poisoned optimization cache is a miss, not a serving failure. + self.state + .lock() + .ok()? + .pages + .get(&NativeProjectionKey::new(oid)) + .cloned() + } + + fn insert(&self, oid: ObjectHash, directory: Arc) { + // Bound retained payload as well as directory count: a wide directory + // retains all entries and codec names even though its root is <=16 KiB. + const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; + let payload_bytes = std::mem::size_of::() + + directory.page_bytes.capacity() + + directory.entries.capacity() * std::mem::size_of::() + + directory.codec_entries.capacity() * std::mem::size_of::() + + directory + .entries + .iter() + .map(|entry| entry.name.capacity() + entry.oid.capacity()) + .sum::() + + directory + .codec_entries + .iter() + .map(|entry| entry.name.capacity()) + .sum::(); + if payload_bytes > MAX_PAYLOAD_BYTES { + return; + } + if let Ok(mut state) = self.state.lock() { + let key = NativeProjectionKey::new(oid); + if state.pages.contains_key(&key) { + return; + } + if state.pages.len() >= 8192 + || state.retained_payload_bytes + payload_bytes > MAX_PAYLOAD_BYTES + { + state.pages.clear(); + state.retained_payload_bytes = 0; + } + state.retained_payload_bytes += payload_bytes; + state.pages.insert(key, directory); + } + } } #[derive(Debug, Clone)] @@ -55,51 +210,82 @@ pub struct DirEntry { } /// Build the MTP2 page for one directory of the fixed view. `rel_path` is -/// scope-relative ("/" = the root directory); `root_tree` is the fixed view's -/// root tree — never a current-ref read. Every entry is represented; +/// an absolute path within the fixed global view, including the descriptor +/// scope prefix ("/" = the global root); `root_tree` is that view's global +/// root tree — never a rebased scope tree or current-ref read. Every entry is represented; /// unsupported entries reject the whole projection (spec: no silent drops). /// -/// Results are memoized per (root tree, path): a page is a pure function of -/// the pinned root tree, and the recursive build otherwise re-walks the same -/// subtree once per ancestor and once per request (a sync over N directories -/// would rebuild the tree N times). Blob sizes/digests are persisted in -/// `mst2_verified_object`, so a cache miss after eviction is a bounded, -/// correctness-identical recomputation. -/// Page memoization table: (root tree id, scope-relative path) → built page. -type PageCache = Mutex>>; - +/// Native results are memoized by storage assembly, tagged subtree tree OID +/// and verification/profile revision. A new global root does not invalidate +/// unchanged subtrees. The fixed root is resolved to `rel_path` once; recursive +/// descent then follows direct child OIDs rather than re-walking from the root. pub async fn build_directory_page( handler: &T, root_tree: &git_internal::internal::object::tree::Tree, rel_path: &str, ) -> Result, SnapshotError> { - static PAGE_CACHE: OnceLock = OnceLock::new(); - // Pages are small (≤16 KiB + entries); 200k pages is far beyond any real - // view. On overflow the cache clears wholesale — a miss only costs a - // rebuild, never correctness. - const PAGE_CACHE_MAX: usize = 200_000; - - let key = (root_tree.id.to_string(), rel_path.to_string()); - let cache = PAGE_CACHE.get_or_init(|| Mutex::new(HashMap::new())); - if let Some(hit) = cache.lock().unwrap().get(&key) { - return Ok(Arc::clone(hit)); + let (directory, work) = build_directory_page_with_work(handler, root_tree, rel_path).await?; + tracing::debug!( + ?work, + path = rel_path, + "native snapshot projection memoization work" + ); + Ok(directory) +} + +/// Build one directory, returning operation-local counters. Authorization and +/// the fixed-view source selection remain the caller's responsibility. +pub async fn build_directory_page_with_work( + handler: &T, + root_tree: &Tree, + rel_path: &str, +) -> Result<(Arc, ProjectionWork), SnapshotError> { + let storage = handler.get_context(); + let mut work = ProjectionWork::default(); + let native = handler.native_snapshot_projection(); + if rel_path == "/" { + // Even cloning a wide fixed root would copy all its entries on a hit. + let directory = + build_subtree(handler, &storage, root_tree, rel_path, native, &mut work).await?; + return Ok((directory, work)); } - let built = Arc::new(build_directory_page_uncached(handler, root_tree, rel_path).await?); - let mut cache = cache.lock().unwrap(); - if cache.len() >= PAGE_CACHE_MAX { - cache.clear(); + let tree = fetch_tree_with_work(handler, root_tree, rel_path, &mut work).await?; + let directory = build_subtree(handler, &storage, &tree, rel_path, native, &mut work).await?; + Ok((directory, work)) +} + +fn cached_subtree( + storage: &Storage, + oid: ObjectHash, + rel_path: &str, + native: bool, + work: &mut ProjectionWork, +) -> Result>, SnapshotError> { + if native && let Some(hit) = storage.native_projection_cache.get(oid) { + hit.path_budget.validate_at(rel_path)?; + work.reused_subtree_roots += 1; + work.directory_root_pages_reused += 1; + work.directory_root_page_bytes_reused += hit.page_bytes.len() as u64; + return Ok(Some(hit)); } - cache.insert(key, Arc::clone(&built)); - Ok(built) + Ok(None) } -async fn build_directory_page_uncached( +async fn build_subtree( handler: &T, - root_tree: &git_internal::internal::object::tree::Tree, + storage: &Storage, + tree: &Tree, rel_path: &str, -) -> Result { - let tree = fetch_tree(handler, root_tree, rel_path).await?; - let dirents = crate::ceres::snapshot::resolver::direct_entries(&tree)?; + native: bool, + work: &mut ProjectionWork, +) -> Result, SnapshotError> { + crate::ceres::snapshot::view::validate_scope_relative_path(rel_path)?; + if let Some(hit) = cached_subtree(storage, tree.id, rel_path, native, work)? { + return Ok(hit); + } + let dirents = crate::ceres::snapshot::resolver::direct_entries(tree)?; + work.directories_rebuilt += 1; + work.directory_entries_scanned += dirents.len() as u64; // T03 write-through verification: consult verified records first, then // fetch + hash the misses and persist them. Invalid records and lookup @@ -109,8 +295,7 @@ async fn build_directory_page_uncached( .filter(|(_, k, _)| *k != FsKind::Directory) .map(|(_, _, oid)| oid.clone()) .collect(); - let verified = handler - .get_context() + let verified = storage .mono_storage() .get_verified_blobs(blob_oids) .await @@ -128,6 +313,7 @@ async fn build_directory_page_uncached( })?; let mut new_verified = HashMap::new(); + let mut path_budget = DescendantPathBudget::default(); let mut entries = Vec::with_capacity(dirents.len()); let mut codec_entries = Vec::with_capacity(dirents.len()); for (name, fs_kind, oid) in dirents { @@ -139,8 +325,36 @@ async fn build_directory_page_uncached( crate::ceres::snapshot::view::validate_scope_relative_path(&child_rel)?; match fs_kind { FsKind::Directory => { - let child_page = - Box::pin(build_directory_page(handler, root_tree, &child_rel)).await?; + let child_oid = + ObjectHash::from_hex_for_kind(tree.id.kind(), &oid).map_err(|e| { + SnapshotError::new(SnapshotErrorCode::IntegrityError, e.to_string()) + })?; + let child_page = if let Some(hit) = + cached_subtree(storage, child_oid, &child_rel, native, work)? + { + hit + } else { + work.tree_fetches += 1; + let child_tree = handler.get_tree_by_hash(&oid).await.map_err(|e| { + SnapshotError::new(SnapshotErrorCode::Internal, e.to_string()) + })?; + if child_tree.id != child_oid { + return Err(SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "fetched child tree identity mismatch", + )); + } + Box::pin(build_subtree( + handler, + storage, + &child_tree, + &child_rel, + native, + work, + )) + .await? + }; + path_budget.include(&name, child_page.path_budget); codec_entries.push(Entry::dir(name.as_bytes(), child_page.page_id)); entries.push(DirEntry { name, @@ -152,7 +366,9 @@ async fn build_directory_page_uncached( }); } FsKind::Regular | FsKind::Executable | FsKind::Symlink => { + path_budget.include(&name, DescendantPathBudget::default()); let (size, digest) = if let Some(v) = verified.get(&oid) { + work.verified_blob_hits += 1; // Verified record: 64-bit size + raw digest, no content read. let d: [u8; 32] = v.raw_sha256.as_slice().try_into().map_err(|_| { SnapshotError::new( @@ -168,9 +384,12 @@ async fn build_directory_page_uncached( })?; (size, d) } else { + work.verified_blob_misses += 1; let raw = fetch_raw_blob(handler, &oid).await?; + work.raw_bytes_fetched += raw.len() as u64; let mut h = Sha256::new(); h.update(&raw); + work.raw_bytes_hashed += raw.len() as u64; let digest: [u8; 32] = h.finalize().into(); new_verified.entry(oid.clone()).or_insert( crate::callisto::mst2_verified_object::ActiveModel { @@ -208,8 +427,7 @@ async fn build_directory_page_uncached( if !new_verified.is_empty() { // Records only ever describe already-fetched content; a persist // failure loses an optimization, never correctness. - if let Err(e) = handler - .get_context() + if let Err(e) = storage .mono_storage() .insert_verified_blobs(new_verified.into_values().collect()) .await @@ -224,13 +442,22 @@ async fn build_directory_page_uncached( format!("MTP2 build failed for {rel_path}: {e}"), ) })?; + work.directory_root_pages_built += 1; + work.directory_root_page_bytes_built += page_bytes.len() as u64; let pid = page_id(&page_bytes); - Ok(BuiltDirectory { + let built = Arc::new(BuiltDirectory { page_bytes, page_id: pid, entries, codec_entries, - }) + path_budget, + }); + if native { + storage + .native_projection_cache + .insert(tree.id, Arc::clone(&built)); + } + Ok(built) } /// Proof pages from the scope root down to (and including) `rel_path`, @@ -259,23 +486,24 @@ pub fn base64_of(data: &[u8]) -> String { base64::engine::general_purpose::STANDARD.encode(data) } -async fn fetch_tree( +async fn fetch_tree_with_work( handler: &T, root_tree: &git_internal::internal::object::tree::Tree, rel_path: &str, + work: &mut ProjectionWork, ) -> Result { crate::ceres::snapshot::view::validate_scope_relative_path(rel_path)?; if rel_path == "/" { return Ok(root_tree.clone()); } let comps: Vec<&str> = rel_path[1..].split('/').collect(); - let mut current = root_tree.clone(); - for (i, comp) in comps.iter().enumerate() { - let last = i == comps.len() - 1; - let item = current - .tree_items - .iter() - .find(|x| x.name == *comp) + let mut current = Cow::Borrowed(root_tree); + for comp in comps { + let position = current.tree_items.iter().position(|x| x.name == comp); + work.scope_path_entries_examined += + position.map_or(current.tree_items.len(), |index| index + 1) as u64; + let item = position + .map(|index| ¤t.tree_items[index]) .ok_or_else(|| { SnapshotError::new( SnapshotErrorCode::PathNotFound, @@ -288,30 +516,15 @@ async fn fetch_tree( "gitlink entries are not supported in this profile", ) })?; - if last { - if kind != FsKind::Directory { - return Err(SnapshotError::new( - SnapshotErrorCode::NotDirectory, - format!("{rel_path} is not a directory"), - )); - } - return handler - .get_tree_by_hash(&item.id.to_string()) - .await - .map_err(|e| { - SnapshotError::new( - SnapshotErrorCode::Internal, - format!("tree fetch failed for {rel_path}: {e}"), - ) - }); - } if kind != FsKind::Directory { return Err(SnapshotError::new( SnapshotErrorCode::NotDirectory, - "intermediate component is not a directory", + format!("{rel_path} is not a directory"), )); } - current = handler + let expected_oid = item.id; + work.tree_fetches += 1; + let fetched = handler .get_tree_by_hash(&item.id.to_string()) .await .map_err(|e| { @@ -320,8 +533,15 @@ async fn fetch_tree( format!("tree fetch failed for component '{comp}': {e}"), ) })?; + if fetched.id != expected_oid { + return Err(SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "fetched scope tree identity mismatch", + )); + } + current = Cow::Owned(fetched); } - unreachable!("loop returns on the last component") + Ok(current.into_owned()) } /// Outcome of resolving one absolute view path (spec 04 §7 statuses). @@ -446,6 +666,10 @@ pub fn hex_of(id: &[u8; 32]) -> String { hex(id) } +#[cfg(test)] +#[path = "native_projection_tests.rs"] +mod native_projection_tests; + #[cfg(test)] mod tests { use std::sync::Arc; @@ -829,9 +1053,16 @@ mod tests { ]) .unwrap(); let handler = crate::ceres::api_service::mono_api_service::MonoApiService::from(&state); - let built = build_directory_page_uncached(&handler, &tree, "/") - .await - .unwrap(); + let built = build_subtree( + &handler, + &state.storage, + &tree, + "/", + false, + &mut ProjectionWork::default(), + ) + .await + .unwrap(); assert_eq!(built.entries.len(), 2); assert!(built.entries.iter().all(|entry| entry.size == Some(10) && entry.content_digest == Some(Sha256::digest(raw).into()))); diff --git a/src/jupiter/storage/mod.rs b/src/jupiter/storage/mod.rs index 17121c7d..deb4fe34 100644 --- a/src/jupiter/storage/mod.rs +++ b/src/jupiter/storage/mod.rs @@ -124,6 +124,9 @@ impl AppService { #[derive(Clone)] pub struct Storage { pub(crate) app_service: Arc, + /// Derived native projection memoization, scoped to this storage assembly. + /// Clones share it; independent databases/backends never share entries. + pub(crate) native_projection_cache: Arc, pub cl_service: CLService, pub push_queue_service: PushQueueService, pub artifact_service: ArtifactService, @@ -279,6 +282,7 @@ impl Storage { Ok(Storage { app_service: app_service.into(), + native_projection_cache: Arc::default(), config_handle, config, cl_service: CLService::new(base.clone()), @@ -561,6 +565,7 @@ impl Storage { Storage { app_service, + native_projection_cache: Arc::default(), // app_service: AppService::mock(), cl_service: CLService::mock(), push_queue_service: PushQueueService::new( diff --git a/src/jupiter/tests.rs b/src/jupiter/tests.rs index aab57e64..ed70ed6f 100644 --- a/src/jupiter/tests.rs +++ b/src/jupiter/tests.rs @@ -373,6 +373,7 @@ pub async fn test_storage_with_config(temp_dir: impl AsRef, config: Config Storage { app_service: Arc::new(svc), + native_projection_cache: Arc::default(), cl_service: CLService::mock(), push_queue_service: PushQueueService::new( base.clone(), From 37a599d6af9361eb20ba1735debe99b8cd24e353 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 03:40:47 +0800 Subject: [PATCH 02/14] fix(mst2): validate snapshot settings through ordinary config loading --- src/commands/config.rs | 111 ++++++++++++++++++++++++++++ src/config/validate.rs | 159 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 267 insertions(+), 3 deletions(-) diff --git a/src/commands/config.rs b/src/commands/config.rs index a4e2763a..f08a287d 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1091,6 +1091,117 @@ mod tests { assert_eq!(load_mode(&matches), LoadMode::RawSources); } + #[test] + fn config_validate_accepts_mst2_file_source_and_checks_identity() { + use futures::FutureExt; + + let lock = env_lock(); + // The repository test environment still carries this removed setting. + // Keep the file-source regression isolated for both load and validate. + let _mail = crate::config::testing::EnvVarGuard::remove(&lock, "MEGA_MAIL__ENABLED"); + let _enabled = crate::config::testing::EnvVarGuard::remove(&lock, "MEGA_MST2__ENABLED"); + let _identity = + crate::config::testing::EnvVarGuard::remove(&lock, "MEGA_MST2__INSTANCE_UUID"); + let _publication = + crate::config::testing::EnvVarGuard::remove(&lock, "MEGA_MST2__PUBLICATION_ENABLED"); + let _token = crate::config::testing::EnvVarGuard::remove(&lock, "MEGA_MST2__AUTH_TOKEN"); + let dir = tempfile::tempdir().expect("temp dir"); + let config_path = dir.path().join("config.toml"); + let content = format!( + "{}\n[mst2]\nenabled = true\ninstance_uuid = \"12345678-1234-4234-9234-123456789abc\"\npublication_enabled = false\nauth_token = \"mst2-cli-test-token\"\n", + crate::config::template::config_init_template(dir.path()), + ); + fs::write(&config_path, &content).expect("write MST/2 config"); + let selected = + crate::config::loader::ConfigLoader::new(crate::config::loader::ConfigInput { + cli_path: Some(config_path.clone()), + ..Default::default() + }) + .load_readonly() + .expect("select explicit existing config"); + assert_eq!(selected.path, config_path); + let config = + load_config_for_validate(&selected.path, None).expect("CLI loader accepts MST/2 file"); + assert!(config.mst2.enabled); + assert_eq!( + config.mst2.auth_token.as_deref(), + Some("mst2-cli-test-token") + ); + let invalid_path = dir.path().join("invalid.toml"); + fs::write( + &invalid_path, + content.replace( + "12345678-1234-4234-9234-123456789abc", + "00000000-0000-0000-0000-000000000000", + ), + ) + .expect("write nil UUID config"); + let invalid_config = load_config_for_validate(&invalid_path, None) + .expect("syntactically valid MST/2 file loads before semantic validation"); + validate_config( + &config, + Some(&config_path), + None, + false, + false, + false, + "human", + ) + .now_or_never() + .expect("validation without secret resolution must complete synchronously") + .expect("ordinary CLI validation accepts MST/2"); + let message = validate_config( + &invalid_config, + Some(&invalid_path), + None, + false, + false, + false, + "human", + ) + .now_or_never() + .expect("validation without secret resolution must complete synchronously") + .expect_err("ordinary CLI validation must reject nil UUID") + .to_string(); + assert!(message.contains("mst2.instance_uuid"), "{message}"); + assert!(!message.contains("mst2-cli-test-token"), "{message}"); + } + + #[test] + fn config_validate_rejects_unknown_mst2_profile_field() { + let dir = tempfile::tempdir().expect("temp dir"); + let config_path = dir.path().join("config.toml"); + let profile_path = dir.path().join("config.test.toml"); + fs::write( + &config_path, + crate::config::template::config_init_template(dir.path()), + ) + .expect("write base config"); + fs::write( + &profile_path, + "[mst2]\nenabled = false\npublication_enabeld = true\n", + ) + .expect("write typo profile"); + let selected = + crate::config::loader::ConfigLoader::new(crate::config::loader::ConfigInput { + cli_path: Some(config_path), + cli_profile: Some("test".to_string()), + ..Default::default() + }) + .load_readonly() + .expect("select explicit existing profile"); + let message = load_config_for_validate( + &selected.path, + selected + .profile + .as_ref() + .map(|profile| profile.path.as_path()), + ) + .expect_err("MST/2 nested typo must fail in the CLI profile loader") + .to_string(); + assert!(message.contains("mst2.publication_enabeld"), "{message}"); + } + #[test] fn config_validate_accepts_deny_warnings_flag() { let matches = cli() diff --git a/src/config/validate.rs b/src/config/validate.rs index a7cb9b88..7804ff4d 100644 --- a/src/config/validate.rs +++ b/src/config/validate.rs @@ -9,8 +9,8 @@ use url::Url; use super::{ ArtifactGcConfig, BlameConfig, BuckConfig, CedarConfig, Config, DbConfig, GitConfig, - GithubSyncConfig, LFSConfig, LogConfig, MonoConfig, NotificationConfig, OAuthConfig, - PackConfig, PushAuth, PushPolicy, RedisConfig, VAULT_AUDIT_SINKS, VaultConfig, + GithubSyncConfig, LFSConfig, LogConfig, MonoConfig, Mst2Config, NotificationConfig, + OAuthConfig, PackConfig, PushAuth, PushPolicy, RedisConfig, VAULT_AUDIT_SINKS, VaultConfig, normalize_token_path, secret::{SecretRef, SecretResolver, is_secret_ref_value}, }; @@ -123,6 +123,7 @@ impl Config { validate_lfs_config(&self.lfs)?; validate_redis_config(&self.redis)?; validate_cedar_config(&self.cedar)?; + validate_mst2_config(&self.mst2)?; if let Some(buck_config) = &self.buck { validate_buck_config(buck_config)?; } @@ -148,6 +149,34 @@ impl Config { } } +fn validate_mst2_config(config: &Mst2Config) -> Result<(), MegaError> { + if !config.enabled { + return Ok(()); + } + let instance_uuid = config.instance_uuid.as_deref().ok_or_else(|| { + MegaError::Other("mst2.instance_uuid is required when mst2.enabled is true".to_string()) + })?; + let instance_uuid = uuid::Uuid::parse_str(instance_uuid).map_err(|_| { + MegaError::Other("mst2.instance_uuid must be a valid non-nil UUID".to_string()) + })?; + if instance_uuid.is_nil() { + return Err(MegaError::Other( + "mst2.instance_uuid must be a valid non-nil UUID".to_string(), + )); + } + if config + .auth_token + .as_deref() + .is_some_and(|token| token.trim().is_empty()) + { + return Err(MegaError::Other( + "mst2.auth_token must not be empty when configured and mst2.enabled is true" + .to_string(), + )); + } + Ok(()) +} + /// Validate `[oauth]` settings: website API base URL, session cookie names, and /// each CORS origin must be a browser Origin of the form `scheme://host[:port]` /// (http/https, no path/query/fragment) that also parses as an HTTP header value @@ -2032,6 +2061,7 @@ pub(crate) fn known_fields(path: &str) -> Option<&'static [&'static str]> { "object_storage", "oauth", "blame", + "mst2", "redis", "buck", "artifacts_gc", @@ -2095,6 +2125,12 @@ pub(crate) fn known_fields(path: &str) -> Option<&'static [&'static str]> { "enable_caching", ]), "redis" => Some(&["url"]), + "mst2" => Some(&[ + "enabled", + "instance_uuid", + "publication_enabled", + "auth_token", + ]), "buck" => Some(&[ "session_timeout", "max_file_size", @@ -2190,7 +2226,7 @@ mod tests { StorageEventsTargetConfig, secret::{SecretRef, SecretResolver}, template::config_init_template, - testing::isolated_config, + testing::{EnvVarGuard, env_lock, isolated_config}, }; #[rustfmt::skip] use crate::orbit_api::factory::{GcsConfig, LocalConfig, ObjectStorageBackend, S3Config}; @@ -2199,6 +2235,123 @@ mod tests { isolated_config(std::env::temp_dir().join("mega2-config-validate-tests")) } + #[test] + fn config_validate_mst2_checks_enabled_identity_and_configured_token() { + let mut config = valid_config(); + config.mst2.enabled = true; + for instance_uuid in [ + None, + Some(""), + Some("invalid-uuid"), + Some("00000000-0000-0000-0000-000000000000"), + ] { + config.mst2.instance_uuid = instance_uuid.map(str::to_owned); + config.mst2.auth_token = Some("mst2-private-token-must-not-appear".to_string()); + let message = config + .validate() + .expect_err("enabled MST/2 requires a non-nil UUID") + .to_string(); + assert!(message.contains("mst2.instance_uuid"), "{message}"); + assert!( + !message.contains("mst2-private-token-must-not-appear"), + "{message}" + ); + } + config.mst2.instance_uuid = Some("12345678-1234-4234-9234-123456789abc".to_string()); + for token in ["", " \t\n"] { + config.mst2.auth_token = Some(token.to_string()); + let message = config + .validate() + .expect_err("configured MST/2 token must not be blank") + .to_string(); + assert!(message.contains("mst2.auth_token"), "{message}"); + } + config.mst2.auth_token = None; + config + .validate() + .expect("existing unauthenticated lab mode remains valid"); + config.mst2.auth_token = Some("mst2-test-token".to_string()); + config.mst2.publication_enabled = true; + config + .validate() + .expect("authenticated publication config is valid"); + config.mst2.enabled = false; + config.mst2.publication_enabled = false; + config.mst2.instance_uuid = None; + config.mst2.auth_token = None; + config + .validate() + .expect("disabled MST/2 defaults remain valid"); + } + + #[test] + fn config_load_str_accepts_mst2_and_expands_file_token_without_reexpansion() { + let lock = env_lock(); + let _enabled = EnvVarGuard::remove(&lock, "MEGA_MST2__ENABLED"); + let _identity = EnvVarGuard::remove(&lock, "MEGA_MST2__INSTANCE_UUID"); + let _publication = EnvVarGuard::remove(&lock, "MEGA_MST2__PUBLICATION_ENABLED"); + let _token = EnvVarGuard::remove(&lock, "MEGA_MST2__AUTH_TOKEN"); + let dir = tempfile::tempdir().expect("temp dir"); + let token_file = dir.path().join("mst2-token"); + let token = "literal-${must_not_be_reexpanded}"; + std::fs::write(&token_file, format!("{token}\n")).expect("write private test token"); + let file_placeholder = format!("${{file:{}}}", token_file.display()); + let content = format!( + "{}\n[mst2]\nenabled = true\ninstance_uuid = \"12345678-1234-4234-9234-123456789abc\"\npublication_enabled = false\nauth_token = {}\n", + config_init_template(dir.path()), + toml::Value::String(file_placeholder), + ); + let loaded = + Config::load_str(&content).expect("ordinary config loader must recognize MST/2"); + assert!(loaded.mst2.enabled); + assert!(!loaded.mst2.publication_enabled); + assert_eq!(loaded.mst2.auth_token.as_deref(), Some(token)); + loaded + .validate() + .expect("expanded MST/2 config must validate"); + assert!( + known_unconsumed_fields(&toml::from_str(&content).expect("parse source")).is_empty() + ); + for field in [ + "enabled", + "instance_uuid", + "publication_enabled", + "auth_token", + ] { + assert!(is_known_field_path(&format!("mst2.{field}"))); + } + assert!(!is_known_field_path("mst2.auth_token.extra")); + assert!(!is_known_field_path("mst2.typo")); + std::fs::write(&token_file, "\n").expect("write empty private test token"); + let empty_token = + Config::load_str(&content).expect("file token still uses ordinary expansion"); + let message = empty_token + .validate() + .expect_err("expanded empty token must fail semantic validation") + .to_string(); + assert!(message.contains("mst2.auth_token"), "{message}"); + } + + #[test] + fn config_load_str_rejects_mst2_typo_and_unknown_nested_table() { + let content = r#" + [mst2] + enabled = false + publication_enabeld = true + [mst2.credentials] + auth_token = "private-value-must-not-appear" + "#; + let message = Config::load_str(content) + .expect_err("MST/2 unknown fields must fail closed") + .to_string(); + assert!(message.contains("mst2.publication_enabeld"), "{message}"); + assert!(message.contains("mst2.credentials"), "{message}"); + assert!( + !message.contains("private-value-must-not-appear"), + "{message}" + ); + } + #[test] fn config_validate_accepts_default_isolated_test_config() { valid_config() From d3471f6f11547699b3e896d4ea6f785a06a34c1c Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 06:44:31 +0800 Subject: [PATCH 03/14] feat(mst2): persist publication request receipts --- src/callisto/mod.rs | 1 + src/callisto/mst2_publication.rs | 6 +- src/callisto/mst2_queue_noop_receipt.rs | 34 + ...100_add_mst2_publication_request_digest.rs | 48 + src/jupiter/migration/mod.rs | 7 +- src/jupiter/service/push_queue_service.rs | 646 +++++++++- src/jupiter/storage/mod.rs | 1 + src/jupiter/storage/mono_storage.rs | 122 +- .../storage/mst2_publication_storage.rs | 505 ++++++++ src/jupiter/storage/mst2_publication_tests.rs | 1080 +++++++++++++++++ src/jupiter/storage/push_queue_storage.rs | 43 +- 11 files changed, 2336 insertions(+), 157 deletions(-) create mode 100644 src/callisto/mst2_queue_noop_receipt.rs create mode 100644 src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs create mode 100644 src/jupiter/storage/mst2_publication_storage.rs create mode 100644 src/jupiter/storage/mst2_publication_tests.rs diff --git a/src/callisto/mod.rs b/src/callisto/mod.rs index 3285765e..38dcb52b 100644 --- a/src/callisto/mod.rs +++ b/src/callisto/mod.rs @@ -60,6 +60,7 @@ pub mod mega_webhook_delivery; pub mod mega_webhook_event_type; pub mod mst2_publication; pub mod mst2_publication_outbox; +pub mod mst2_queue_noop_receipt; pub mod mst2_verified_object; pub mod notification_event_types; pub mod oci_blob_ref; diff --git a/src/callisto/mst2_publication.rs b/src/callisto/mst2_publication.rs index ff69518e..88549464 100644 --- a/src/callisto/mst2_publication.rs +++ b/src/callisto/mst2_publication.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; pub struct Model { #[sea_orm(primary_key, auto_increment = true)] pub id: i64, - /// Deterministic writer-side operation id (e.g. push old→new). + /// Server-assigned writer identity (e.g. `mst2:trunk-queue:`). pub operation_id: String, /// Namespace path this publication advances (e.g. "/"). pub namespace: String, @@ -29,6 +29,10 @@ pub struct Model { pub writer_epoch: i64, /// `trunk_push` | `web_edit` | `import` | … (spec 09 §5 writer matrix). pub writer_kind: String, + /// Missing on legacy receipts; never reconstructed from current refs. + #[sea_orm(column_type = "Text", nullable)] + pub request_digest: Option, + pub request_digest_version: Option, pub created_at: DateTimeWithTimeZone, } diff --git a/src/callisto/mst2_queue_noop_receipt.rs b/src/callisto/mst2_queue_noop_receipt.rs new file mode 100644 index 00000000..15c3b137 --- /dev/null +++ b/src/callisto/mst2_queue_noop_receipt.rs @@ -0,0 +1,34 @@ +//! Committed trunk-queue no-op results, separate from visible publications. + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_queue_noop_receipt")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true)] + pub id: i64, + #[sea_orm(column_type = "Text")] + pub operation_id: String, + #[sea_orm(column_type = "Text")] + pub namespace: String, + #[sea_orm(column_type = "Text")] + pub request_digest: String, + pub request_digest_version: i32, + pub writer_epoch: i64, + #[sea_orm(column_type = "Text")] + pub writer_kind: String, + pub observed_sequence: i64, + #[sea_orm(column_type = "Text")] + pub observed_root_commit: String, + #[sea_orm(column_type = "Text")] + pub observed_root_tree: String, + #[sea_orm(column_type = "Text")] + pub landed_commit_id: String, + pub created_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs b/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs new file mode 100644 index 00000000..8186b7c4 --- /dev/null +++ b/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs @@ -0,0 +1,48 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared( + r#"ALTER TABLE mst2_publication + ADD COLUMN IF NOT EXISTS request_digest text, + ADD COLUMN IF NOT EXISTS request_digest_version integer; + DO $$ BEGIN + ALTER TABLE mst2_publication + ADD CONSTRAINT mst2_publication_request_digest_valid CHECK ( + (request_digest IS NULL AND request_digest_version IS NULL) OR + (request_digest IS NOT NULL AND request_digest_version IS NOT NULL + AND request_digest_version > 0 + AND request_digest ~ '^sha256:[0-9a-f]{64}$') + ); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + CREATE TABLE IF NOT EXISTS mst2_queue_noop_receipt ( + id bigserial PRIMARY KEY, + operation_id text NOT NULL UNIQUE, + namespace text NOT NULL, + request_digest text NOT NULL CHECK (request_digest ~ '^sha256:[0-9a-f]{64}$'), + request_digest_version integer NOT NULL CHECK (request_digest_version > 0), + writer_epoch bigint NOT NULL, + writer_kind text NOT NULL, + observed_sequence bigint NOT NULL CHECK (observed_sequence >= 0), + observed_root_commit text NOT NULL, + observed_root_tree text NOT NULL, + landed_commit_id text NOT NULL, + created_at timestamptz NOT NULL + );"#, + ) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Receipts remain immutable, including the absence of legacy digests. + Ok(()) + } +} diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 822a1f72..54e795f3 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -139,6 +139,7 @@ mod m20260921_000100_fix_git_tag_unique; mod m20260923_000100_import_repo_cleanups; mod m20260923_000200_canonicalize_import_repo_paths; mod m20260925_000100_media_paging; +mod m20261005_000100_add_mst2_publication_request_digest; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -266,6 +267,7 @@ impl MigratorTrait for Migrator { Box::new(m20260923_000100_import_repo_cleanups::Migration), Box::new(m20260923_000200_canonicalize_import_repo_paths::Migration), Box::new(m20260925_000100_media_paging::Migration), + Box::new(m20261005_000100_add_mst2_publication_request_digest::Migration), ] } } @@ -475,13 +477,14 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 3..], + &names[names.len() - 4..], &[ "m20260923_000100_import_repo_cleanups".to_string(), "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), + "m20261005_000100_add_mst2_publication_request_digest".to_string(), ], - "media paging registered last, after the FU-14/16 migrations" + "MST/2 publication digest migration follows the FU-14/16 migrations" ); let db = alias_db().await; diff --git a/src/jupiter/service/push_queue_service.rs b/src/jupiter/service/push_queue_service.rs index 9d563268..dfa5a5f3 100644 --- a/src/jupiter/service/push_queue_service.rs +++ b/src/jupiter/service/push_queue_service.rs @@ -29,6 +29,10 @@ use crate::{ blob_path_index::BlobPathIndexMode, git_db_storage::GitDbStorage, mono_storage::MonoStorage, + mst2_publication_storage::{ + PreparedPublication, PublicationPreparation, PublicationReceiptError, + PublicationRequest, + }, push_queue_storage::{ ClaimOutcome, EnqueueOutcome, EnqueueParams, EnqueueRejectReason, PushQueueStorage, }, @@ -416,7 +420,7 @@ pub struct PushExecContext { pub storage: crate::jupiter::storage::Storage, pub git_object_cache: std::sync::Arc, /// Test-only: two-phase sync inside the B3 txn right before - /// `apply_push_in_txn` (after the baseline read): the executor signals + /// `apply_push_in_txn` or the n=0 root CAS (after the baseline read): the executor signals /// "race window open" on the enter barrier, then blocks on the release /// barrier until the test's queue-bypassing writer has committed — a /// deterministic apply-time root CAS miss (WH-03). @@ -425,6 +429,12 @@ pub struct PushExecContext { pub pre_apply_release_barrier: Option>, } +struct PushRefBaseline<'a> { + cur_commit: Option<&'a str>, + cur_tree: Option<&'a str>, + root: Option<&'a crate::callisto::mega_refs::Model>, +} + /// Context required to execute a claimed `kind=merge` round under B3. pub struct MergeExecContext { pub storage: crate::jupiter::storage::Storage, @@ -1264,6 +1274,51 @@ impl PushQueueService { return Ok(ExecuteOutcome::HardStopped { id: req.id }); } + // Receipt replay precedes baseline/ref writes: the root may already + // have advanced since a committed operation lost its response. + let publication = if let Some(ctx) = push_ctx + && ctx.storage.config().mst2.publication_enabled + && row.kind != PushQueueKindEnum::Attach + && !(row.kind == PushQueueKindEnum::Merge && merge_ctx.is_some()) + { + let request = match PublicationRequest::from_trunk_queue(&row) { + Ok(request) => request, + Err(error) => { + return self + .b3_fail_merge(txn, req.id, "Conflict", error.to_string()) + .await; + } + }; + let preparation = match self + .mono_storage + .begin_publication_in_txn(&txn, request) + .await + { + Ok(preparation) => preparation, + Err(PublicationReceiptError::Database(error)) => return Err(error.into()), + Err(error) => { + return self + .b3_fail_merge(txn, req.id, "Conflict", error.to_string()) + .await; + } + }; + match preparation { + PublicationPreparation::Prepared(prepared) => Some(prepared), + PublicationPreparation::AlreadyCommitted(committed) => { + return self + .b3_replay_committed_operation(txn, req.id, committed.receipt.new_oid) + .await; + } + PublicationPreparation::AlreadyCommittedNoop(receipt) => { + return self + .b3_replay_committed_operation(txn, req.id, receipt.landed_commit_id) + .await; + } + } + } else { + None + }; + // Expected-root baseline compare (NULL-safe). let root = self.mono_storage.get_main_ref_in_txn("/", &txn).await?; let (cur_commit, cur_tree) = match &root { @@ -1346,7 +1401,17 @@ impl PushQueueService { } if let Some(ctx) = push_ctx { return self - .b3_execute_push(txn, &row, ctx, cur_commit, cur_tree, root.as_ref()) + .b3_execute_push( + txn, + &row, + ctx, + PushRefBaseline { + cur_commit, + cur_tree, + root: root.as_ref(), + }, + publication, + ) .await; } tracing::trace!(policy = ?self.push_policy, id = req.id, "B3 push stub"); @@ -1449,26 +1514,16 @@ impl PushQueueService { return Ok(self.outcome_claim_lost(req.id)); } - // T05 (spec 09 §1): record the publication receipt, outbox event and - // per-namespace sequence in the *same* transaction as the root CAS - // above, so the visible change, its sequence and its outbox commit - // atomically. Gated off by default (`mst2.publication_enabled`), so - // an unconfigured deployment keeps the exact prior behavior. - if let Some(ctx) = push_ctx - && ctx.storage.config().mst2.publication_enabled - { - let old_oid = cur_commit.unwrap_or(ZERO_ID); - let namespace = self.mono_storage.normalize_namespace(&row.path); + if let Some(prepared) = publication { self.mono_storage .record_publication_in_txn( &txn, - &row.operation_id, - &namespace, - old_oid, + prepared, + cur_commit.unwrap_or(ZERO_ID), &new_commit, - "trunk_push", ) - .await?; + .await + .map_err(|error| MegaError::Other(error.to_string()))?; } PushQueueStorage::notify_mono_write_queue(&txn).await?; @@ -1481,6 +1536,25 @@ impl PushQueueService { }) } + async fn b3_replay_committed_operation( + &self, + txn: sea_orm::DatabaseTransaction, + id: i64, + landed_commit_id: String, + ) -> Result { + if !PushQueueStorage::mark_done_if_running_in_txn(&txn, id, &landed_commit_id).await? { + txn.rollback().await?; + return Ok(self.outcome_claim_lost(id)); + } + PushQueueStorage::notify_mono_write_queue(&txn).await?; + txn.commit().await?; + Ok(ExecuteOutcome::Done { + id, + landed_commit_id, + root_cas_writes: 0, + }) + } + /// TP-08: attach kind under held `MONO_WRITE_LOCK` — sole root write is /// `attach_to_monorepo_parent_in_txn` (no generic CAS stub, no retry loop). /// @@ -2308,13 +2382,12 @@ impl PushQueueService { txn: sea_orm::DatabaseTransaction, row: &push_queue::Model, ctx: &PushExecContext, - cur_commit: Option<&str>, - cur_tree: Option<&str>, - root: Option<&crate::callisto::mega_refs::Model>, + baseline: PushRefBaseline<'_>, + publication: Option, ) -> Result { let id = row.id; match self - .b3_execute_push_inner(txn, row, ctx, cur_commit, cur_tree, root) + .b3_execute_push_inner(txn, row, ctx, baseline, publication) .await { Ok(outcome) => Ok(outcome), @@ -2357,10 +2430,14 @@ impl PushQueueService { txn: sea_orm::DatabaseTransaction, row: &push_queue::Model, ctx: &PushExecContext, - cur_commit: Option<&str>, - cur_tree: Option<&str>, - root: Option<&crate::callisto::mega_refs::Model>, + baseline: PushRefBaseline<'_>, + publication: Option, ) -> Result { + let PushRefBaseline { + cur_commit, + cur_tree, + root, + } = baseline; use std::path::PathBuf; use git_internal::{ @@ -2481,6 +2558,12 @@ impl PushQueueService { ) .await; } + if let Some(barrier) = &ctx.pre_apply_enter_barrier { + barrier.wait().await; + } + if let Some(barrier) = &ctx.pre_apply_release_barrier { + barrier.wait().await; + } PushQueueStorage::savepoint(&txn, "b3_kind").await?; let cas_ok = self .mono_storage @@ -2517,6 +2600,18 @@ impl PushQueueService { txn.rollback().await?; return Ok(self.outcome_claim_lost(row.id)); } + if let Some(prepared) = publication { + self.mono_storage + .record_noop_operation_in_txn( + &txn, + prepared, + &root.ref_commit_hash, + &root.ref_tree_hash, + &pref.ref_commit_hash, + ) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } PushQueueStorage::notify_mono_write_queue(&txn).await?; txn.commit().await?; self.run_c_segment_index(row.id, &row.path).await; @@ -2786,26 +2881,16 @@ impl PushQueueService { return Ok(self.outcome_claim_lost(row.id)); } - // T05 (spec 09 §1/§7): record the publication receipt, outbox event - // and per-namespace sequence in the *same* transaction as the ref CAS - // above, so the visible change, its sequence and its outbox commit - // atomically (trunk push = the primary default-namespace writer of - // spec 09 §5). Gated off by default (`mst2.publication_enabled`): - // a deployment that has not passed the §9 shadow comparison keeps - // the exact prior behavior. Replaying the same operation id lands - // the receipt once (PUB-11). - if ctx.storage.config().mst2.publication_enabled { - let namespace = self.mono_storage.normalize_namespace(&normalized); + if let Some(prepared) = publication { self.mono_storage .record_publication_in_txn( &txn, - &row.operation_id, - &namespace, + prepared, cur_commit.unwrap_or(ZERO_ID), &landed_commit_id, - "trunk_push", ) - .await?; + .await + .map_err(|error| MegaError::Other(error.to_string()))?; } PushQueueStorage::notify_mono_write_queue(&txn).await?; @@ -4804,6 +4889,491 @@ mod tests { .unwrap() } + #[tokio::test] + async fn mst2_receipt_real_push_replays_before_root_baseline_and_rejects_changed_request() { + use sea_orm::{ + ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, Statement, + }; + + use crate::callisto::{mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt}; + + let temp = tempfile::tempdir().unwrap(); + let mut config = crate::config::testing::isolated_config(temp.path().join("config")); + config.monorepo.push_policy = PushPolicy::Trunk; + config.mst2.publication_enabled = true; + let storage = crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; + let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; + let (old_commit, path) = wh03_path_fixture(&storage, "receipt").await; + let old_id = old_commit.id.to_string(); + let (new_id, payload) = wh03_save_n1_commit( + &storage, + old_commit.id, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "receipt new", + ) + .await; + let id = wh03_enqueue_push(&storage, &path, &old_id, &new_id, &payload).await; + assert!(matches!( + wh03_exec(&storage, id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let mono = storage.mono_storage(); + let root_after = mono.get_main_ref("/").await.unwrap().unwrap(); + let operation_id = format!("mst2:trunk-queue:{id}"); + let receipt = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(&operation_id)) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.request_digest_version, Some(1)); + assert!(receipt.request_digest.is_some()); + assert_eq!(receipt.new_oid, new_id); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + let retry = |requester: Option, payload: JsonValue| EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: push_operation_id(&old_id, &new_id), + path: path.clone(), + old_id: old_id.clone(), + new_id: new_id.clone(), + requester, + payload, + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }; + assert!( + matches!(storage.push_queue_service.enqueue(retry(None, payload.to_json())).await.unwrap(), + EnqueueOutcome::Replay { id: replay_id, .. } if replay_id == id) + ); + let error = storage + .push_queue_service + .enqueue(retry(Some("different-actor".to_owned()), payload.to_json())) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + let mut different = payload.to_json(); + different["fork_base"] = serde_json::json!("different-base"); + let error = storage + .push_queue_service + .enqueue(retry(None, different)) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + + // Model recovery of a Running queue row after COMMIT lost its response. + // Original claim baselines remain old, so a late replay would fail them. + mono.get_connection() + .execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', landed_commit_id = NULL WHERE id = $1", + [id.into()], + )) + .await + .unwrap(); + assert_eq!( + wh03_exec(&storage, id).await, + ExecuteOutcome::Done { + id, + landed_commit_id: new_id.clone(), + root_cas_writes: 0, + } + ); + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!(root.ref_commit_hash, root_after.ref_commit_hash); + assert_eq!(root.ref_tree_hash, root_after.ref_tree_hash); + let check_txn = mono.get_connection().begin().await.unwrap(); + assert!( + !PushQueueStorage::is_hard_stopped_in_txn(&check_txn) + .await + .unwrap() + ); + check_txn.rollback().await.unwrap(); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', requester = 'changed-actor' WHERE id = $1", + [id.into()], + )).await.unwrap(); + assert!( + matches!(wh03_exec(&storage, id).await, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("MST2_PUBLICATION_CONFLICT")) + ); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + root_after.ref_commit_hash + ); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + // Distinct internal no-op operations do not create publications. + let n0_payload = PushPayload { + commits: Vec::new(), + fork_base: None, + n: 0, + }; + let mut noop_ids = Vec::new(); + for round in 0..3 { + let operation_id = format!("internal-noop-{round}"); + let outcome = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: operation_id.clone(), + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: None, + payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap(); + let EnqueueOutcome::Inserted { id: n0_id } = outcome else { + panic!("{outcome:?}"); + }; + assert_eq!( + storage + .push_queue_service + .storage() + .claim_for_execution(n0_id) + .await + .unwrap(), + ClaimOutcome::Claimed + ); + assert!(matches!( + wh03_exec(&storage, n0_id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let noop = mst2_queue_noop_receipt::Entity::find() + .filter( + mst2_queue_noop_receipt::Column::OperationId + .eq(format!("mst2:trunk-queue:{n0_id}")), + ) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(noop.observed_sequence, 1); + assert_eq!(noop.observed_root_commit, root_after.ref_commit_hash); + assert_eq!(noop.observed_root_tree, root_after.ref_tree_hash); + assert_eq!(noop.landed_commit_id, new_id); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + assert!(matches!(storage.push_queue_service.enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, operation_id: operation_id.clone(), + path: path.clone(), old_id: new_id.clone(), new_id: new_id.clone(), + requester: None, payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), is_delete: false, + }).await.unwrap(), EnqueueOutcome::Replay { id: replay_id, .. } if replay_id == n0_id)); + let error = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: operation_id.clone(), + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: Some("changed-actor".to_owned()), + payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + let mut changed_payload = n0_payload.to_json(); + changed_payload["fork_base"] = serde_json::json!("changed-intent"); + let error = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id, + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: None, + payload: changed_payload, + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + noop_ids.push(n0_id); + } + let (later_id, later_payload) = wh03_save_n1_commit( + &storage, + new_id.parse().unwrap(), + "ffffffffffffffffffffffffffffffffffffffff", + "after no-op", + ) + .await; + let later_queue_id = + wh03_enqueue_push(&storage, &path, &new_id, &later_id, &later_payload).await; + assert!(matches!( + wh03_exec(&storage, later_queue_id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let later_root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_ne!(later_root.ref_commit_hash, root_after.ref_commit_hash); + for &n0_id in &noop_ids { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', landed_commit_id = NULL WHERE id = $1", + [n0_id.into()], + )).await.unwrap(); + assert_eq!( + wh03_exec(&storage, n0_id).await, + ExecuteOutcome::Done { + id: n0_id, + landed_commit_id: new_id.clone(), + root_cas_writes: 0, + } + ); + } + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!(root.ref_commit_hash, later_root.ref_commit_hash); + assert_eq!(root.ref_tree_hash, later_root.ref_tree_hash); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 2); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 3 + ); + let n0_id = noop_ids[0]; + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', requester = 'changed-actor' WHERE id = $1", + [n0_id.into()], + )).await.unwrap(); + assert!( + matches!(wh03_exec(&storage, n0_id).await, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("MST2_PUBLICATION_CONFLICT")) + ); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + later_root.ref_commit_hash + ); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 2); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 3 + ); + } + + #[tokio::test] + async fn mst2_noop_real_push_checks_epoch_baseline_and_net_zero_cas_fences() { + use sea_orm::{ConnectionTrait, EntityTrait, PaginatorTrait, Statement}; + + use crate::callisto::{mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt}; + + for fence in ["epoch", "baseline", "cas"] { + let temp = tempfile::tempdir().unwrap(); + let mut config = crate::config::testing::isolated_config(temp.path().join("config")); + config.monorepo.push_policy = PushPolicy::Trunk; + config.mst2.publication_enabled = true; + let storage = + crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; + let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; + let (tip, path) = wh03_path_fixture(&storage, &format!("noop-{fence}")).await; + let tip_id = tip.id.to_string(); + let payload = PushPayload { + commits: Vec::new(), + fork_base: None, + n: 0, + }; + let id = wh03_enqueue_push(&storage, &path, &tip_id, &tip_id, &payload).await; + let mono = storage.mono_storage(); + let root_before = mono.get_main_ref("/").await.unwrap().unwrap(); + let outcome = if fence == "epoch" { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ($1, 0, 2)", + [path.clone().into()], + )).await.unwrap(); + wh03_exec(&storage, id).await + } else if fence == "baseline" { + mono.get_connection() + .execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET expected_commit_hash = $1 WHERE id = $2", + ["f".repeat(40).into(), id.into()], + )) + .await + .unwrap(); + wh03_exec(&storage, id).await + } else { + let enter = Arc::new(tokio::sync::Barrier::new(2)); + let release = Arc::new(tokio::sync::Barrier::new(2)); + let ctx = PushExecContext { + storage: storage.clone(), + git_object_cache: Arc::new(crate::ceres::api_service::cache::GitObjectCache { + connection: crate::jupiter::tests::test_redis_manager().await, + prefix: String::new(), + }), + pre_apply_enter_barrier: Some(Arc::clone(&enter)), + pre_apply_release_barrier: Some(Arc::clone(&release)), + }; + let exec_storage = storage.clone(); + let mut worker = tokio::spawn(async move { + exec_storage + .push_queue_service + .execute_b3( + ExecuteRequest { + id, + ..Default::default() + }, + None, + None, + Some(&ctx), + ) + .await + }); + let handshake = async { + tokio::time::timeout(Duration::from_secs(5), enter.wait()).await.map_err(|_| "no-op did not reach CAS".to_owned())?; + tokio::time::timeout(Duration::from_secs(5), mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE mega_refs SET ref_commit_hash = $1 WHERE path = '/' AND ref_name = $2", + ["f".repeat(40).into(), MEGA_BRANCH_NAME.into()], + ))).await.map_err(|_| "bypass writer timed out".to_owned())?.map_err(|error| error.to_string())?; + tokio::time::timeout(Duration::from_secs(5), release.wait()).await.map_err(|_| "no-op CAS release timed out".to_owned()) + }.await; + if let Err(error) = handshake { + worker.abort(); + let _ = tokio::time::timeout(Duration::from_secs(5), &mut worker).await; + panic!("{error}"); + } + match tokio::time::timeout(Duration::from_secs(10), &mut worker).await { + Ok(result) => result.unwrap().unwrap(), + Err(_) => { + worker.abort(); + let _ = tokio::time::timeout(Duration::from_secs(5), &mut worker).await; + panic!("no-op CAS worker did not finish"); + } + } + }; + if fence == "epoch" { + assert!( + matches!(outcome, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("writer epoch is fenced")) + ); + } else { + assert_eq!(outcome, ExecuteOutcome::BypassDetected { id }); + } + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!( + root.ref_commit_hash, + if fence == "cas" { + "f".repeat(40) + } else { + root_before.ref_commit_hash + } + ); + assert_eq!(root.ref_tree_hash, root_before.ref_tree_hash); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 0); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + let txn = mono.get_connection().begin().await.unwrap(); + assert_eq!( + PushQueueStorage::is_hard_stopped_in_txn(&txn) + .await + .unwrap(), + fence != "epoch" + ); + txn.rollback().await.unwrap(); + } + } + /// Same as `wh03_exec`, with an explicit request override (force flags). async fn wh03_exec_with( storage: &crate::jupiter::storage::Storage, diff --git a/src/jupiter/storage/mod.rs b/src/jupiter/storage/mod.rs index 17121c7d..2c750f97 100644 --- a/src/jupiter/storage/mod.rs +++ b/src/jupiter/storage/mod.rs @@ -16,6 +16,7 @@ pub mod issue_storage; pub mod lfs_db_storage; pub mod media_paging_storage; pub mod mono_storage; +pub(crate) mod mst2_publication_storage; pub mod notification_storage; pub mod object_storage; pub mod oci_db_storage; diff --git a/src/jupiter/storage/mono_storage.rs b/src/jupiter/storage/mono_storage.rs index dc140b2e..254b7bd5 100644 --- a/src/jupiter/storage/mono_storage.rs +++ b/src/jupiter/storage/mono_storage.rs @@ -26,7 +26,7 @@ use sea_orm::{ use crate::{ callisto::{ mega_blob, mega_cl, mega_commit, mega_ref_tombstones, mega_refs, mega_tag, mega_tree, - mst2_publication, mst2_publication_outbox, mst2_verified_object, + mst2_verified_object, }, common::{ errors::MegaError, @@ -1830,112 +1830,6 @@ impl MonoStorage { } } - /// Record one namespace publication inside the caller's transaction - /// (spec 09 §1/§7): bump the per-namespace sequence, insert the - /// receipt and append the outbox event — atomically with the ref CAS - /// the same transaction performs. - /// - /// Idempotent per (namespace, operation_id): a retried writer gets - /// back the original sequence and neither the counter nor the outbox - /// advances twice (PUB-11). The uniqueness is scoped to the namespace - /// because push operation ids (`old→new`) are only unique within one - /// repo: two namespaces landing the same (old,new) pair are distinct - /// publications, not replays. - pub async fn record_publication_in_txn( - &self, - txn: &DatabaseTransaction, - operation_id: &str, - namespace: &str, - old_oid: &str, - new_oid: &str, - writer_kind: &str, - ) -> Result { - // One SELECT by operation id answers both questions: the same id - // under this namespace is a replay (PUB-11) and returns the original - // sequence; the same id under a different namespace means the id is - // not actually unique to this publication — refuse rather than - // silently double-bind it. - if let Some(existing) = mst2_publication::Entity::find() - .filter(mst2_publication::Column::OperationId.eq(operation_id)) - .one(txn) - .await? - { - if existing.namespace == namespace { - return Ok(existing.sequence); - } - return Err(MegaError::Other(format!( - "operation id {operation_id} already published under a different namespace" - ))); - } - // Upsert the sequence row and bump it atomically; the unique - // namespace key makes this a single-winner counter. - let backend = txn.get_database_backend(); - let bump = sea_orm::Statement::from_sql_and_values( - backend, - r#"INSERT INTO mst2_namespace_seq ("namespace", "sequence", "epoch") - VALUES ($1, 1, 1) - ON CONFLICT ("namespace") DO UPDATE SET "sequence" = "mst2_namespace_seq"."sequence" + 1 - RETURNING "sequence""#, - [namespace.into()], - ); - let row = txn - .query_one_raw(bump) - .await? - .ok_or_else(|| MegaError::Other("mst2_namespace_seq upsert returned no row".into()))?; - let sequence: i64 = row.try_get_by_index(0)?; - let now = chrono::Utc::now().fixed_offset(); - - // Concurrency-safe insert: the receipt is unique per - // (namespace, operation_id). The replay check above covers a - // committed prior publication; a truly concurrent same-operation - // writer loses at the outbox insert's unique operation_id below, - // rolling this transaction back rather than double-publishing. - let insert = mst2_publication::ActiveModel { - id: sea_orm::ActiveValue::NotSet, - operation_id: Set(operation_id.to_string()), - namespace: Set(namespace.to_string()), - sequence: Set(sequence), - old_oid: Set(old_oid.to_string()), - new_oid: Set(new_oid.to_string()), - writer_epoch: Set(1), - writer_kind: Set(writer_kind.to_string()), - created_at: Set(now), - }; - match mst2_publication::Entity::insert(insert) - .on_conflict( - OnConflict::columns([ - mst2_publication::Column::Namespace, - mst2_publication::Column::OperationId, - ]) - .do_nothing() - .to_owned(), - ) - .exec(txn) - .await - { - // RecordNotInserted means a receipt with this identity already - // exists; see the outbox insert below for how a racing writer - // of the same operation is resolved. - Ok(_) | Err(DbErr::RecordNotInserted) => {} - Err(e) => return Err(e.into()), - } - - // The replay check above already returned for a pre-existing receipt, - // so reaching here means this call owns `sequence`; the outbox row - // inserts normally. - mst2_publication_outbox::ActiveModel { - id: sea_orm::ActiveValue::NotSet, - operation_id: Set(operation_id.to_string()), - namespace: Set(namespace.to_string()), - sequence: Set(sequence), - state: Set("PENDING".to_string()), - created_at: Set(now), - } - .insert(txn) - .await?; - Ok(sequence) - } - /// Canonical publication namespace for a repository path: a leading /// slash, no trailing slash, root is `/`. Publication identities are /// keyed on this so `/p/` and `/p` never count as two namespaces. @@ -3261,7 +3155,7 @@ mod tests { // Rollback ⇒ nothing visible: the receipt commits only with the // ref CAS in the same transaction (spec 09 §1). let txn = mono.get_connection().begin().await.unwrap(); - mono.record_publication_in_txn(&txn, "op-roll", "/", "", "aaa", "trunk_push") + mono.record_test_publication_in_txn(&txn, "op-roll", "/", "", "aaa", "trunk_push") .await .unwrap(); txn.rollback().await.unwrap(); @@ -3281,7 +3175,7 @@ mod tests { for (op, old, new) in [("op-1", "", "a1"), ("op-2", "a1", "a2")] { let txn = mono.get_connection().begin().await.unwrap(); let seq = mono - .record_publication_in_txn(&txn, op, "/", old, new, "trunk_push") + .record_test_publication_in_txn(&txn, op, "/", old, new, "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3300,7 +3194,7 @@ mod tests { // adds nothing (PUB-11) — the push retry path depends on this. let txn = mono.get_connection().begin().await.unwrap(); let again = mono - .record_publication_in_txn(&txn, "op-1", "/", "", "a1", "trunk_push") + .record_test_publication_in_txn(&txn, "op-1", "/", "", "a1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3317,7 +3211,7 @@ mod tests { // Distinct namespaces keep independent sequences. let txn = mono.get_connection().begin().await.unwrap(); let other = mono - .record_publication_in_txn(&txn, "op-child", "/child", "", "c1", "trunk_push") + .record_test_publication_in_txn(&txn, "op-child", "/child", "", "c1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3334,13 +3228,13 @@ mod tests { // replay: refuse rather than silently double-bind the id (the // reviewer's P3.1 — a silent no-op left /b's counter at 0 forever). let txn = mono.get_connection().begin().await.unwrap(); - mono.record_publication_in_txn(&txn, "shared-op", "/", "", "a1", "trunk_push") + mono.record_test_publication_in_txn(&txn, "shared-op", "/", "", "a1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); let txn = mono.get_connection().begin().await.unwrap(); let err = mono - .record_publication_in_txn(&txn, "shared-op", "/b", "", "a1", "trunk_push") + .record_test_publication_in_txn(&txn, "shared-op", "/b", "", "a1", "trunk_push") .await; assert!(err.is_err(), "cross-namespace reuse must be refused"); drop(txn); // the failed transaction rolls back on drop @@ -3363,7 +3257,7 @@ mod tests { let m = base.mono_storage(); let txn = m.get_connection().begin().await.unwrap(); let seq = m - .record_publication_in_txn( + .record_test_publication_in_txn( &txn, &format!("op-c{i}"), "/", diff --git a/src/jupiter/storage/mst2_publication_storage.rs b/src/jupiter/storage/mst2_publication_storage.rs new file mode 100644 index 00000000..dacc3850 --- /dev/null +++ b/src/jupiter/storage/mst2_publication_storage.rs @@ -0,0 +1,505 @@ +//! SQL receipt identity for the existing trunk queue adapter. +//! +//! This is not a complete namespace/projection publication implementation. + +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, ConnectionTrait, DatabaseTransaction, DbErr, + EntityTrait, QueryFilter, Statement, +}; +use sha2::{Digest, Sha256}; + +use crate::{ + callisto::{ + mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt, push_queue, + sea_orm_active_enums::PushQueueKindEnum, + }, + common::utils::MEGA_BRANCH_NAME, + jupiter::storage::mono_storage::MonoStorage, +}; + +const REQUEST_DIGEST_VERSION: i32 = 1; +const WRITER_EPOCH: i64 = 1; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PublicationReceiptError { + #[error("MST2_PUBLICATION_CONFLICT: {0}")] + Conflict(String), + #[error("MST2_PUBLICATION_LEGACY_RECEIPT: operation {0} has no supported request digest")] + LegacyReceipt(String), + #[error("MST2_PUBLICATION_INTEGRITY_ERROR: {0}")] + Integrity(String), + #[error(transparent)] + Database(#[from] DbErr), +} + +/// Constructed from a server-persisted queue row, never a client digest. +#[derive(Debug, Clone)] +pub(crate) struct PublicationRequest { + operation_id: String, + namespace: String, + writer_kind: String, + request_digest: String, + legacy_operation_id: Option, + is_noop: bool, +} + +impl PublicationRequest { + pub(crate) fn from_trunk_queue( + row: &push_queue::Model, + ) -> Result { + if row.id <= 0 || row.operation_id.is_empty() { + return Err(PublicationReceiptError::Integrity( + "invalid queue operation identity".into(), + )); + } + mst2_codec::descriptor::validate_scope(&row.path) + .map_err(|error| PublicationReceiptError::Integrity(error.to_string()))?; + let writer_kind = match &row.kind { + PushQueueKindEnum::Push => "trunk_push", + PushQueueKindEnum::Merge => "trunk_merge_stub", + PushQueueKindEnum::Attach => { + return Err(PublicationReceiptError::Integrity( + "attach is not covered by this adapter".into(), + )); + } + }; + let operation_id = format!("mst2:trunk-queue:{}", row.id); + let mut hash = request_hasher(&operation_id, &row.path, writer_kind); + hash_field(&mut hash, row.operation_id.as_bytes()); + hash_field(&mut hash, MEGA_BRANCH_NAME.as_bytes()); + hash_field(&mut hash, row.old_id.as_bytes()); + hash_field(&mut hash, row.new_id.as_bytes()); + hash.update([u8::from(row.requester.is_some())]); + if let Some(actor) = &row.requester { + hash_field(&mut hash, actor.as_bytes()); + } + hash_json(&mut hash, &row.payload); + Ok(Self { + operation_id, + namespace: row.path.clone(), + writer_kind: writer_kind.to_string(), + request_digest: format!("sha256:{}", hex::encode(hash.finalize())), + legacy_operation_id: Some(row.operation_id.clone()), + is_noop: row.kind == PushQueueKindEnum::Push + && row.old_id == row.new_id + && row.payload.get("n").and_then(serde_json::Value::as_u64) == Some(0), + }) + } + + #[cfg(test)] + fn for_test( + operation_id: &str, + namespace: &str, + old: &str, + new: &str, + writer_kind: &str, + ) -> Self { + let mut hash = request_hasher(operation_id, namespace, writer_kind); + hash_field(&mut hash, old.as_bytes()); + hash_field(&mut hash, new.as_bytes()); + Self { + operation_id: operation_id.to_owned(), + namespace: namespace.to_owned(), + writer_kind: writer_kind.to_owned(), + request_digest: format!("sha256:{}", hex::encode(hash.finalize())), + legacy_operation_id: None, + is_noop: false, + } + } +} + +fn request_hasher(operation_id: &str, namespace: &str, writer_kind: &str) -> Sha256 { + let mut hash = Sha256::new(); + hash.update(b"mega.mst2.trunk-queue-request\0"); + hash.update(REQUEST_DIGEST_VERSION.to_le_bytes()); + hash.update(WRITER_EPOCH.to_le_bytes()); + hash_field(&mut hash, operation_id.as_bytes()); + hash_field(&mut hash, namespace.as_bytes()); + hash_field(&mut hash, writer_kind.as_bytes()); + hash +} + +fn hash_field(hash: &mut Sha256, bytes: &[u8]) { + hash.update((bytes.len() as u64).to_le_bytes()); + hash.update(bytes); +} + +fn hash_json(hash: &mut Sha256, value: &serde_json::Value) { + use serde_json::Value; + match value { + Value::Null => hash.update([0]), + Value::Bool(value) => hash.update([1, u8::from(*value)]), + Value::Number(value) => { + hash.update([2]); + hash_field(hash, value.to_string().as_bytes()); + } + Value::String(value) => { + hash.update([3]); + hash_field(hash, value.as_bytes()); + } + Value::Array(values) => { + hash.update([4]); + hash.update((values.len() as u64).to_le_bytes()); + for value in values { + hash_json(hash, value); + } + } + Value::Object(values) => { + hash.update([5]); + hash.update((values.len() as u64).to_le_bytes()); + let mut keys: Vec<_> = values.keys().collect(); + keys.sort_unstable(); + for key in keys { + hash_field(hash, key.as_bytes()); + hash_json(hash, &values[key]); + } + } + } +} + +#[derive(Debug)] +pub(crate) struct CommittedPublication { + pub(crate) receipt: mst2_publication::Model, + pub(crate) outbox: mst2_publication_outbox::Model, +} + +#[derive(Debug)] +pub(crate) enum PublicationPreparation { + Prepared(PreparedPublication), + AlreadyCommitted(CommittedPublication), + AlreadyCommittedNoop(mst2_queue_noop_receipt::Model), +} + +/// The SQL transaction id prevents finalizing a reservation in another txn. +#[derive(Debug)] +pub(crate) struct PreparedPublication { + request: PublicationRequest, + sequence: i64, + transaction_id: i64, +} + +async fn validate_committed_publication( + txn: &DatabaseTransaction, + request: &PublicationRequest, + receipt: mst2_publication::Model, +) -> Result { + if receipt.namespace != request.namespace { + return Err(PublicationReceiptError::Conflict( + "operation belongs to another namespace".into(), + )); + } + if receipt.request_digest_version != Some(REQUEST_DIGEST_VERSION) + || receipt.request_digest.is_none() + { + return Err(PublicationReceiptError::LegacyReceipt( + request.operation_id.clone(), + )); + } + if receipt.writer_kind != request.writer_kind + || receipt.writer_epoch != WRITER_EPOCH + || receipt.request_digest.as_deref() != Some(request.request_digest.as_str()) + { + return Err(PublicationReceiptError::Conflict( + "operation request changed".into(), + )); + } + let outbox = mst2_publication_outbox::Entity::find() + .filter(mst2_publication_outbox::Column::OperationId.eq(&request.operation_id)) + .one(txn) + .await? + .ok_or_else(|| PublicationReceiptError::Integrity("receipt outbox missing".into()))?; + if outbox.namespace != receipt.namespace || outbox.sequence != receipt.sequence { + return Err(PublicationReceiptError::Integrity( + "receipt outbox identity mismatch".into(), + )); + } + Ok(CommittedPublication { receipt, outbox }) +} + +async fn reject_legacy_queue_receipt( + txn: &DatabaseTransaction, + legacy_operation_id: Option<&str>, + namespace: &str, +) -> Result<(), PublicationReceiptError> { + if let Some(legacy_id) = legacy_operation_id + && let Some(legacy) = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(legacy_id)) + .filter(mst2_publication::Column::Namespace.eq(namespace)) + .one(txn) + .await? + && (legacy.request_digest_version != Some(REQUEST_DIGEST_VERSION) + || legacy.request_digest.is_none()) + { + return Err(PublicationReceiptError::LegacyReceipt(legacy_id.to_owned())); + } + Ok(()) +} + +enum RecordedOperation { + Publication(mst2_publication::Model), + Noop(mst2_queue_noop_receipt::Model), +} + +async fn find_committed_operation( + txn: &DatabaseTransaction, + operation_id: &str, +) -> Result, PublicationReceiptError> { + let publication = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(operation_id)) + .one(txn) + .await?; + let noop = mst2_queue_noop_receipt::Entity::find() + .filter(mst2_queue_noop_receipt::Column::OperationId.eq(operation_id)) + .one(txn) + .await?; + match (publication, noop) { + (Some(_), Some(_)) => Err(PublicationReceiptError::Integrity( + "operation has both publication and no-op receipts".into(), + )), + (Some(receipt), None) => Ok(Some(RecordedOperation::Publication(receipt))), + (None, Some(receipt)) => Ok(Some(RecordedOperation::Noop(receipt))), + (None, None) => Ok(None), + } +} + +async fn validate_committed_operation( + txn: &DatabaseTransaction, + request: &PublicationRequest, + existing: RecordedOperation, +) -> Result { + match existing { + RecordedOperation::Publication(receipt) => Ok(PublicationPreparation::AlreadyCommitted( + validate_committed_publication(txn, request, receipt).await?, + )), + RecordedOperation::Noop(receipt) => { + if receipt.namespace != request.namespace + || receipt.writer_kind != request.writer_kind + || receipt.writer_epoch != WRITER_EPOCH + || !request.is_noop + || receipt.request_digest != request.request_digest + { + return Err(PublicationReceiptError::Conflict( + "operation request changed".into(), + )); + } + if receipt.request_digest_version != REQUEST_DIGEST_VERSION { + return Err(PublicationReceiptError::LegacyReceipt( + request.operation_id.clone(), + )); + } + if mst2_publication_outbox::Entity::find() + .filter(mst2_publication_outbox::Column::OperationId.eq(&request.operation_id)) + .one(txn) + .await? + .is_some() + { + return Err(PublicationReceiptError::Integrity( + "no-op operation has a publication outbox".into(), + )); + } + Ok(PublicationPreparation::AlreadyCommittedNoop(receipt)) + } + } +} + +impl MonoStorage { + /// Queue admission can replay Done without executing B3; audit that path too. + pub(crate) async fn validate_queue_publication_replay_in_txn( + &self, + txn: &DatabaseTransaction, + candidate: &push_queue::Model, + ) -> Result<(), PublicationReceiptError> { + let operation_id = format!("mst2:trunk-queue:{}", candidate.id); + let Some(existing) = find_committed_operation(txn, &operation_id).await? else { + reject_legacy_queue_receipt(txn, Some(&candidate.operation_id), &candidate.path) + .await?; + return Ok(()); + }; + let request = PublicationRequest::from_trunk_queue(candidate)?; + validate_committed_operation(txn, &request, existing).await?; + Ok(()) + } + + /// Lock before any business ref write, then recheck immutable receipt identity. + pub(crate) async fn begin_publication_in_txn( + &self, + txn: &DatabaseTransaction, + request: PublicationRequest, + ) -> Result { + txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ($1, 0, $2) \ + ON CONFLICT (namespace) DO NOTHING", + [request.namespace.clone().into(), WRITER_EPOCH.into()], + )) + .await?; + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT sequence, epoch, txid_current() AS transaction_id \ + FROM mst2_namespace_seq WHERE namespace = $1 FOR UPDATE", + [request.namespace.clone().into()], + )) + .await? + .ok_or_else(|| PublicationReceiptError::Integrity("namespace row missing".into()))?; + let sequence: i64 = row.try_get("", "sequence")?; + let epoch: i64 = row.try_get("", "epoch")?; + if let Some(existing) = find_committed_operation(txn, &request.operation_id).await? { + return validate_committed_operation(txn, &request, existing).await; + } + if epoch != WRITER_EPOCH || sequence < 0 { + return Err(PublicationReceiptError::Conflict( + "writer epoch is fenced".into(), + )); + } + reject_legacy_queue_receipt( + txn, + request.legacy_operation_id.as_deref(), + &request.namespace, + ) + .await?; + Ok(PublicationPreparation::Prepared(PreparedPublication { + request, + sequence, + transaction_id: row.try_get("", "transaction_id")?, + })) + } + + /// Finalize only the request reserved before this transaction's business writes. + pub(crate) async fn record_publication_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedPublication, + old_oid: &str, + new_oid: &str, + ) -> Result { + let request = prepared.request; + if request.is_noop { + return Err(PublicationReceiptError::Integrity( + "no-op requests do not publish".into(), + )); + } + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE mst2_namespace_seq SET sequence = sequence + 1 \ + WHERE namespace = $1 AND sequence = $2 AND epoch = $3 AND txid_current() = $4 \ + RETURNING sequence", + [ + request.namespace.clone().into(), + prepared.sequence.into(), + WRITER_EPOCH.into(), + prepared.transaction_id.into(), + ], + )) + .await? + .ok_or_else(|| { + PublicationReceiptError::Conflict( + "publication reservation is stale or belongs to another transaction".into(), + ) + })?; + let sequence: i64 = row.try_get("", "sequence")?; + let now = chrono::Utc::now().fixed_offset(); + let receipt = mst2_publication::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id.clone()), + namespace: Set(request.namespace.clone()), + sequence: Set(sequence), + old_oid: Set(old_oid.to_owned()), + new_oid: Set(new_oid.to_owned()), + writer_epoch: Set(WRITER_EPOCH), + writer_kind: Set(request.writer_kind), + request_digest: Set(Some(request.request_digest)), + request_digest_version: Set(Some(REQUEST_DIGEST_VERSION)), + created_at: Set(now), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("receipt-written"); + let outbox = mst2_publication_outbox::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id), + namespace: Set(request.namespace), + sequence: Set(sequence), + state: Set("PENDING".to_owned()), + created_at: Set(now), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("outbox-written"); + Ok(CommittedPublication { receipt, outbox }) + } + + pub(crate) async fn record_noop_operation_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedPublication, + root_commit: &str, + root_tree: &str, + landed_commit_id: &str, + ) -> Result { + if !prepared.request.is_noop { + return Err(PublicationReceiptError::Integrity( + "request is not a trunk push no-op".into(), + )); + } + txn.query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT sequence FROM mst2_namespace_seq \ + WHERE namespace = $1 AND sequence = $2 AND epoch = $3 AND txid_current() = $4 FOR UPDATE", + [prepared.request.namespace.clone().into(), prepared.sequence.into(), WRITER_EPOCH.into(), prepared.transaction_id.into()], + )).await?.ok_or_else(|| PublicationReceiptError::Conflict("no-op reservation is stale or belongs to another transaction".into()))?; + let request = prepared.request; + let receipt = mst2_queue_noop_receipt::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id), + namespace: Set(request.namespace), + request_digest: Set(request.request_digest), + request_digest_version: Set(REQUEST_DIGEST_VERSION), + writer_epoch: Set(WRITER_EPOCH), + writer_kind: Set(request.writer_kind), + observed_sequence: Set(prepared.sequence), + observed_root_commit: Set(root_commit.to_owned()), + observed_root_tree: Set(root_tree.to_owned()), + landed_commit_id: Set(landed_commit_id.to_owned()), + created_at: Set(chrono::Utc::now().fixed_offset()), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("noop-receipt-written"); + Ok(receipt) + } + + #[cfg(test)] + pub(crate) async fn record_test_publication_in_txn( + &self, + txn: &DatabaseTransaction, + operation_id: &str, + namespace: &str, + old: &str, + new: &str, + writer_kind: &str, + ) -> Result { + let request = PublicationRequest::for_test(operation_id, namespace, old, new, writer_kind); + let committed = match self.begin_publication_in_txn(txn, request).await? { + PublicationPreparation::AlreadyCommitted(committed) => committed, + PublicationPreparation::AlreadyCommittedNoop(_) => { + return Err(PublicationReceiptError::Integrity( + "test publication replayed a no-op".into(), + )); + } + PublicationPreparation::Prepared(prepared) => { + self.record_publication_in_txn(txn, prepared, old, new) + .await? + } + }; + Ok(committed.receipt.sequence) + } +} + +#[cfg(test)] +#[path = "mst2_publication_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/mst2_publication_tests.rs b/src/jupiter/storage/mst2_publication_tests.rs new file mode 100644 index 00000000..bb743a44 --- /dev/null +++ b/src/jupiter/storage/mst2_publication_tests.rs @@ -0,0 +1,1080 @@ +use std::sync::Arc; + +use sea_orm::{PaginatorTrait, TransactionTrait}; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::jupiter::{ + migration::{Migrator, apply_migrations}, + storage::{ + base_storage::{BaseStorage, StorageConnector}, + push_queue_storage::{ClaimOutcome, EnqueueOutcome, EnqueueParams, PushQueueStorage}, + }, + tests::test_db_connection, +}; + +fn queue_row() -> push_queue::Model { + let now = chrono::Utc::now().fixed_offset(); + push_queue::Model { + id: 7, + kind: PushQueueKindEnum::Push, + operation_id: "old→new".to_owned(), + path: "/project".to_owned(), + old_id: "old".to_owned(), + new_id: "new".to_owned(), + payload: serde_json::json!({"commits": ["new"], "n": 1, "fork_base": "old"}), + landed_commit_id: None, + status: crate::callisto::sea_orm_active_enums::PushQueueStatusEnum::Running, + requester: Some("alice".to_owned()), + failure_type: None, + error_message: None, + heartbeat_at: now, + superseded_by: None, + expected_commit_hash: Some("root-before".to_owned()), + expected_tree_hash: Some("tree-before".to_owned()), + pending_action: None, + enqueued_at: now, + started_at: Some(now), + finished_at: None, + updated_at: now, + } +} + +fn noop_queue_row() -> push_queue::Model { + let mut row = queue_row(); + row.old_id = "tip".to_owned(); + row.new_id = "tip".to_owned(); + row.payload = serde_json::json!({"commits": [], "fork_base": null, "n": 0}); + row +} + +#[test] +fn queue_digest_is_versioned_and_binds_trusted_intent_not_retry_baseline() { + let row = queue_row(); + let original = PublicationRequest::from_trunk_queue(&row).unwrap(); + assert_eq!(original.operation_id, "mst2:trunk-queue:7"); + let mut retry = row.clone(); + retry.expected_commit_hash = Some("later-root".to_owned()); + retry.expected_tree_hash = Some("later-tree".to_owned()); + retry.started_at = None; + retry.payload = serde_json::from_str(r#"{"n":1,"fork_base":"old","commits":["new"]}"#).unwrap(); + assert_eq!( + PublicationRequest::from_trunk_queue(&retry) + .unwrap() + .request_digest, + original.request_digest + ); + for change in ["actor", "path", "old", "new", "payload", "identity"] { + let mut changed = row.clone(); + match change { + "actor" => changed.requester = Some("bob".to_owned()), + "path" => changed.path = "/other".to_owned(), + "old" => changed.old_id = "other-old".to_owned(), + "new" => changed.new_id = "other-new".to_owned(), + "payload" => changed.payload["commits"] = serde_json::json!(["new", "extra"]), + "identity" => changed.id += 1, + _ => unreachable!(), + } + assert_ne!( + PublicationRequest::from_trunk_queue(&changed) + .unwrap() + .request_digest, + original.request_digest, + "{change}" + ); + } +} + +async fn storage() -> (tempfile::TempDir, MonoStorage) { + let temp = tempfile::tempdir().unwrap(); + let db = test_db_connection(temp.path()).await; + apply_migrations(&db, false).await.unwrap(); + ( + temp, + MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }, + ) +} + +async fn seed_root(mono: &MonoStorage) { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mega_refs (id, path, ref_name, ref_commit_hash, ref_tree_hash, created_at, updated_at, is_cl) \ + VALUES (1, '/', $1, $2, $3, now(), now(), false)", + [MEGA_BRANCH_NAME.into(), "a".repeat(40).into(), "b".repeat(40).into()], + )).await.unwrap(); +} + +async fn prepared( + mono: &MonoStorage, + txn: &DatabaseTransaction, + request: PublicationRequest, +) -> PreparedPublication { + match mono.begin_publication_in_txn(txn, request).await.unwrap() { + PublicationPreparation::Prepared(prepared) => prepared, + PublicationPreparation::AlreadyCommitted(_) => panic!("fresh operation"), + PublicationPreparation::AlreadyCommittedNoop(_) => panic!("fresh no-op operation"), + } +} + +async fn advance(mono: &MonoStorage, txn: &DatabaseTransaction) -> bool { + mono.cas_update_root_main_ref_in_txn( + txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"c".repeat(40), + &"d".repeat(40), + ) + .await + .unwrap() +} + +async fn counts(mono: &MonoStorage) -> (u64, u64) { + ( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + ) +} + +#[tokio::test] +async fn merge_stub_admission_replays_same_receipt_and_rejects_changed_intent() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let queue = PushQueueStorage::new(mono.base.clone()); + let payload = serde_json::json!({"steps": ["first"], "mode": "stub"}); + let admitted = queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("alice"), + payload: payload.clone(), + }) + .await + .unwrap(); + let EnqueueOutcome::Inserted { id } = admitted else { + panic!("{admitted:?}"); + }; + // With no committed receipt, admission retains the existing adoption + // behavior and the original persisted intent remains the execution input. + assert_eq!( + queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("legacy-retry"), + payload: serde_json::json!({"different": "no receipt yet"}), + }) + .await + .unwrap(), + EnqueueOutcome::Adopted { id } + ); + assert_eq!( + queue.claim_for_execution(id).await.unwrap(), + ClaimOutcome::Claimed + ); + let row = push_queue::Entity::find_by_id(id) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(row.requester.as_deref(), Some("alice")); + assert_eq!(row.payload, payload); + let txn = mono.get_connection().begin().await.unwrap(); + let reservation = prepared( + &mono, + &txn, + PublicationRequest::from_trunk_queue(&row).unwrap(), + ) + .await; + assert!(advance(&mono, &txn).await); + let landed = "c".repeat(40); + assert!( + PushQueueStorage::mark_done_if_running_in_txn(&txn, id, &landed) + .await + .unwrap() + ); + let original = mono + .record_publication_in_txn(&txn, reservation, &"a".repeat(40), &landed) + .await + .unwrap(); + assert_eq!(original.receipt.writer_kind, "trunk_merge_stub"); + txn.commit().await.unwrap(); + assert_eq!( + queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("alice"), + payload: payload.clone(), + }) + .await + .unwrap(), + EnqueueOutcome::Replay { + id, + landed_commit_id: Some(landed.clone()) + } + ); + for change in ["actor", "payload"] { + let (requester, retry_payload) = if change == "actor" { + ("mallory", payload.clone()) + } else { + ( + "alice", + serde_json::json!({"steps": ["first", "changed"], "mode": "stub"}), + ) + }; + let error = queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some(requester), + payload: retry_payload, + }) + .await + .unwrap_err(); + assert!( + error.to_string().contains("MST2_PUBLICATION_CONFLICT"), + "{change}" + ); + } + assert_eq!( + mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + vec![original.receipt] + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + vec![original.outbox] + ); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 1); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + landed + ); +} + +#[tokio::test] +async fn same_request_concurrently_returns_one_original_receipt_and_outbox() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let request = PublicationRequest::for_test("same-op", "/", "a", "c", "test"); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for _ in 0..2 { + let mono = mono.clone(); + let request = request.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let (committed, wrote) = + match mono.begin_publication_in_txn(&txn, request).await.unwrap() { + PublicationPreparation::AlreadyCommitted(committed) => (committed, false), + PublicationPreparation::AlreadyCommittedNoop(_) => { + panic!("publication request") + } + PublicationPreparation::Prepared(prepared) => { + assert!(advance(&mono, &txn).await); + ( + mono.record_publication_in_txn(&txn, prepared, "a", "c") + .await + .unwrap(), + true, + ) + } + }; + txn.commit().await.unwrap(); + (committed, wrote) + })); + } + let mut results = Vec::new(); + for worker in workers { + results.push( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(results.iter().filter(|(_, wrote)| *wrote).count(), 1); + assert_eq!(results[0].0.receipt, results[1].0.receipt); + assert_eq!(results[0].0.outbox, results[1].0.outbox); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 1); + assert_eq!(counts(&mono).await, (1, 1)); +} + +#[tokio::test] +async fn changed_request_conflicts_before_ref_writes() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let txn = mono.get_connection().begin().await.unwrap(); + let request = PublicationRequest::from_trunk_queue(&queue_row()).unwrap(); + let reserved = prepared(&mono, &txn, request).await; + assert!(advance(&mono, &txn).await); + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + for change in ["actor", "new", "payload"] { + let mut changed = queue_row(); + match change { + "actor" => changed.requester = Some("mallory".to_owned()), + "new" => changed.new_id = "overwrite".to_owned(), + "payload" => changed.payload["commits"] = serde_json::json!(["injected"]), + _ => unreachable!(), + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&changed).unwrap() + ) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "c".repeat(40) + ); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 1); + assert_eq!(counts(&mono).await, (1, 1)); + } +} + +#[tokio::test] +async fn different_operations_compete_on_actual_root_cas_without_extra_sequence() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for i in 0..2 { + let mono = mono.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let request = + PublicationRequest::for_test(&format!("competitor-{i}"), "/", "a", "c", "test"); + let reserved = prepared(&mono, &txn, request).await; + if !advance(&mono, &txn).await { + txn.rollback().await.unwrap(); + return false; + } + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + true + })); + } + let mut winners = 0; + for worker in workers { + winners += usize::from( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(winners, 1); + assert_eq!(counts(&mono).await, (1, 1)); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 1); +} + +#[tokio::test] +async fn reservation_cannot_be_finalized_from_another_transaction() { + let (_temp, mono) = storage().await; + // Keep the namespace/sequence unchanged after rollback: only the txn id differs. + mono.get_connection() + .execute_unprepared( + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ('/', 0, 1)", + ) + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared( + &mono, + &txn, + PublicationRequest::for_test("txn-op", "/", "a", "c", "test"), + ) + .await; + txn.rollback().await.unwrap(); + let other = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.record_publication_in_txn(&other, reserved, "a", "c") + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + other.rollback().await.unwrap(); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 0); +} + +#[tokio::test] +async fn legacy_receipt_and_queue_alias_do_not_guess_request_digest() { + let (_temp, mono) = storage().await; + let row = queue_row(); + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ($1, $2, 1, 'old', 'new', 1, 'trunk_push', now())", + [row.operation_id.clone().into(), row.path.clone().into()], + )).await.unwrap(); + for request in [ + PublicationRequest::for_test(&row.operation_id, &row.path, "old", "new", "trunk_push"), + PublicationRequest::from_trunk_queue(&row).unwrap(), + ] { + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn(&txn, request).await, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + txn.rollback().await.unwrap(); + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + txn.rollback().await.unwrap(); + let legacy = mst2_publication::Entity::find() + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); + assert!(legacy.request_digest_version.is_none()); + assert_eq!(counts(&mono).await, (1, 0)); +} + +#[tokio::test] +async fn incomplete_outbox_refuses_both_receipt_and_queue_replay() { + for corruption in ["missing", "sequence", "namespace"] { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let row = queue_row(); + let request = PublicationRequest::from_trunk_queue(&row).unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared(&mono, &txn, request.clone()).await; + assert!(advance(&mono, &txn).await); + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + let sql = match corruption { + "missing" => "DELETE FROM mst2_publication_outbox", + "sequence" => "UPDATE mst2_publication_outbox SET sequence = sequence + 1", + "namespace" => "UPDATE mst2_publication_outbox SET namespace = '/other'", + _ => unreachable!(), + }; + mono.get_connection().execute_unprepared(sql).await.unwrap(); + let before = counts(&mono).await; + let txn = mono.get_connection().begin().await.unwrap(); + assert!( + matches!( + mono.begin_publication_in_txn(&txn, request).await, + Err(PublicationReceiptError::Integrity(_)) + ), + "{corruption}" + ); + txn.rollback().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!( + matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::Integrity(_)) + ), + "{corruption}" + ); + txn.rollback().await.unwrap(); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "c".repeat(40) + ); + assert_eq!(mono.publication_sequence(&row.path).await.unwrap(), 1); + assert_eq!(counts(&mono).await, before); + } +} + +#[tokio::test] +async fn concurrent_noop_operations_keep_the_publication_sequence_and_outbox() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for _ in 0..2 { + let mono = mono.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let (receipt, wrote) = match mono + .begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await + .unwrap() + { + PublicationPreparation::Prepared(reserved) => { + assert!( + mono.cas_update_root_main_ref_in_txn( + &txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"a".repeat(40), + &"b".repeat(40) + ) + .await + .unwrap() + ); + ( + mono.record_noop_operation_in_txn( + &txn, + reserved, + &"a".repeat(40), + &"b".repeat(40), + "tip", + ) + .await + .unwrap(), + true, + ) + } + PublicationPreparation::AlreadyCommittedNoop(receipt) => (receipt, false), + PublicationPreparation::AlreadyCommitted(_) => panic!("no-op request"), + }; + txn.commit().await.unwrap(); + (receipt, wrote) + })); + } + let mut results = Vec::new(); + for worker in workers { + results.push( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(results.iter().filter(|(_, wrote)| *wrote).count(), 1); + assert_eq!(results[0].0, results[1].0); + assert_eq!(results[0].0.observed_sequence, 0); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + for change in ["actor", "payload"] { + let mut row = noop_queue_row(); + if change == "actor" { + row.requester = Some("other".to_owned()); + } else { + row.payload["fork_base"] = serde_json::json!("other"); + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&row).unwrap() + ) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + } +} + +#[tokio::test] +async fn noop_receipt_replay_rejects_unsupported_version_and_publication_corruption() { + for corruption in ["version", "publication", "outbox"] { + let (_temp, mono) = storage().await; + let row = noop_queue_row(); + let request = PublicationRequest::from_trunk_queue(&row).unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared(&mono, &txn, request.clone()).await; + mono.record_noop_operation_in_txn(&txn, reserved, "root", "tree", "tip") + .await + .unwrap(); + txn.commit().await.unwrap(); + let sql = match corruption { + "version" => "UPDATE mst2_queue_noop_receipt SET request_digest_version = 2", + "publication" => { + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ('mst2:trunk-queue:7', '/project', 1, 'a', 'b', 1, 'trunk_push', now())" + } + "outbox" => { + "INSERT INTO mst2_publication_outbox (operation_id, namespace, sequence, state, created_at) \ + VALUES ('mst2:trunk-queue:7', '/project', 1, 'PENDING', now())" + } + _ => unreachable!(), + }; + mono.get_connection().execute_unprepared(sql).await.unwrap(); + let publications_before = mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(); + let outbox_before = mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(); + for admission in [false, true] { + let txn = mono.get_connection().begin().await.unwrap(); + let result = if admission { + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await + } else { + mono.begin_publication_in_txn(&txn, request.clone()) + .await + .map(|_| ()) + }; + if corruption == "version" { + assert!(matches!( + result, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + } else { + assert!(matches!(result, Err(PublicationReceiptError::Integrity(_)))); + } + txn.rollback().await.unwrap(); + let namespace = mono + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "SELECT sequence FROM mst2_namespace_seq WHERE namespace = $1", + [row.path.clone().into()], + )) + .await + .unwrap() + .unwrap(); + assert_eq!(namespace.try_get::("", "sequence").unwrap(), 0); + assert_eq!( + mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + publications_before + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + outbox_before + ); + } + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + } +} + +#[tokio::test] +async fn noop_reservation_cannot_move_to_another_transaction() { + let (_temp, mono) = storage().await; + mono.get_connection() + .execute_unprepared( + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ('/project', 0, 1)", + ) + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared( + &mono, + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await; + txn.rollback().await.unwrap(); + let other = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.record_noop_operation_in_txn(&other, reserved, "root", "tree", "tip") + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + other.rollback().await.unwrap(); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn migration_preserves_legacy_receipts_and_validates_digest_columns() { + let temp = tempfile::tempdir().unwrap(); + let db = test_db_connection(temp.path()).await; + let old_count = u32::try_from(Migrator::migrations().len() - 1).unwrap(); + Migrator::up(&db, Some(old_count)).await.unwrap(); + db.execute_unprepared( + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ('legacy', '/', 1, 'a', 'b', 1, 'test', now())" + ).await.unwrap(); + apply_migrations(&db, false).await.unwrap(); + let latest = Migrator::migrations().pop().unwrap(); + latest + .up(&sea_orm_migration::SchemaManager::new(&db)) + .await + .unwrap(); + let legacy = mst2_publication::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); + assert!(legacy.request_digest_version.is_none()); + for (digest, version) in [ + (Some(format!("sha256:{}", "a".repeat(64))), Some(1)), + (Some(format!("sha256:{}", "A".repeat(64))), Some(1)), + (Some("broken".to_owned()), Some(1)), + (None, Some(1)), + (Some(format!("sha256:{}", "a".repeat(64))), None), + (Some(format!("sha256:{}", "a".repeat(64))), Some(0)), + ] { + let valid = digest.as_deref() == Some(format!("sha256:{}", "a".repeat(64)).as_str()) + && version == Some(1); + let txn = db.begin().await.unwrap(); + let result = txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE mst2_publication SET request_digest = $1, request_digest_version = $2 WHERE operation_id = 'legacy'", + [digest.into(), version.into()], + )).await; + assert_eq!(result.is_ok(), valid); + txn.rollback().await.unwrap(); + } + let legacy = mst2_publication::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); +} + +#[cfg(unix)] +pub(super) fn crash_checkpoint(phase: &str) { + if std::env::var("MEGA_MST2_RECEIPT_CRASH_PHASE") + .ok() + .as_deref() + == Some(phase) + { + unsafe { + libc::raise(libc::SIGKILL); + } + panic!("SIGKILL did not terminate the worker"); + } +} + +#[cfg(unix)] +#[test] +fn receipt_crash_worker() { + let Ok(url) = std::env::var("MEGA_MST2_RECEIPT_WORKER_DB") else { + return; + }; + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let db = sea_orm::Database::connect(url).await.unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + let txn = mono.get_connection().begin().await.unwrap(); + let noop = std::env::var("MEGA_MST2_RECEIPT_WORKER_NOOP").is_ok(); + let request = if noop { + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap() + } else { + PublicationRequest::for_test("process-op", "/", "a", "c", "test") + }; + let reserved = prepared(&mono, &txn, request).await; + if noop { + assert!( + mono.cas_update_root_main_ref_in_txn( + &txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"a".repeat(40), + &"b".repeat(40) + ) + .await + .unwrap() + ); + } else { + assert!(advance(&mono, &txn).await); + } + crash_checkpoint("ref-written"); + if noop { + mono.record_noop_operation_in_txn( + &txn, + reserved, + &"a".repeat(40), + &"b".repeat(40), + "tip", + ) + .await + .unwrap(); + } else { + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + } + txn.commit().await.unwrap(); + crash_checkpoint("commit-complete"); + }); +} + +#[cfg(unix)] +#[tokio::test] +async fn sigkill_recovery_and_commit_without_response_use_original_receipt() { + use std::os::unix::process::ExitStatusExt; + + for phase in [ + "ref-written", + "receipt-written", + "outbox-written", + "commit-complete", + ] { + let temp = tempfile::tempdir().unwrap(); + let (config, _schema) = crate::jupiter::tests::test_db_config(temp.path()).await; + let db = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + seed_root(&mono).await; + let mut worker = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "jupiter::storage::mst2_publication_storage::tests::receipt_crash_worker", + "--nocapture", + ]) + .env("MEGA_MST2_RECEIPT_WORKER_DB", &config.db_url) + .env("MEGA_MST2_RECEIPT_CRASH_PHASE", phase) + .env_remove("MEGA_MST2_RECEIPT_WORKER_NOOP") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let status = loop { + if let Some(status) = worker.try_wait().unwrap() { + break status; + } + if std::time::Instant::now() >= deadline { + worker.kill().unwrap(); + worker.wait().unwrap(); + panic!("receipt worker timed out at {phase}"); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }; + assert_eq!(status.signal(), Some(libc::SIGKILL), "{phase}: {status}"); + let committed = phase == "commit-complete"; + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!( + root.ref_commit_hash, + if committed { "c" } else { "a" }.repeat(40) + ); + assert_eq!(counts(&mono).await, if committed { (1, 1) } else { (0, 0) }); + assert_eq!( + mono.publication_sequence("/").await.unwrap(), + i64::from(committed) + ); + let retry_db = sea_orm::Database::connect(config.db_url.as_str()) + .await + .unwrap(); + let retry_mono = MonoStorage { + base: BaseStorage::new(Arc::new(retry_db)), + }; + let txn = retry_mono.get_connection().begin().await.unwrap(); + let request = PublicationRequest::for_test("process-op", "/", "a", "c", "test"); + match retry_mono + .begin_publication_in_txn(&txn, request) + .await + .unwrap() + { + PublicationPreparation::AlreadyCommitted(original) => { + assert!(committed); + assert_eq!(original.receipt.sequence, 1); + assert_eq!(original.receipt.new_oid, "c"); + assert_eq!(original.outbox.sequence, 1); + txn.commit().await.unwrap(); + } + PublicationPreparation::Prepared(_) => { + assert!(!committed); + txn.rollback().await.unwrap(); + } + PublicationPreparation::AlreadyCommittedNoop(_) => panic!("publication request"), + } + assert_eq!(counts(&mono).await, if committed { (1, 1) } else { (0, 0) }); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn noop_sigkill_and_lost_commit_response_never_create_a_publication() { + use std::os::unix::process::ExitStatusExt; + + for phase in ["ref-written", "noop-receipt-written", "commit-complete"] { + let temp = tempfile::tempdir().unwrap(); + let (config, _schema) = crate::jupiter::tests::test_db_config(temp.path()).await; + let db = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + seed_root(&mono).await; + let mut worker = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "jupiter::storage::mst2_publication_storage::tests::receipt_crash_worker", + "--nocapture", + ]) + .env("MEGA_MST2_RECEIPT_WORKER_DB", &config.db_url) + .env("MEGA_MST2_RECEIPT_CRASH_PHASE", phase) + .env("MEGA_MST2_RECEIPT_WORKER_NOOP", "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let status = loop { + if let Some(status) = worker.try_wait().unwrap() { + break status; + } + if std::time::Instant::now() >= deadline { + worker.kill().unwrap(); + worker.wait().unwrap(); + panic!("no-op receipt worker timed out at {phase}"); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }; + assert_eq!(status.signal(), Some(libc::SIGKILL), "{phase}: {status}"); + let committed = phase == "commit-complete"; + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "a".repeat(40) + ); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + u64::from(committed) + ); + let original = mst2_queue_noop_receipt::Entity::find() + .one(mono.get_connection()) + .await + .unwrap(); + if committed { + let txn = mono.get_connection().begin().await.unwrap(); + assert!(advance(&mono, &txn).await); + txn.commit().await.unwrap(); + } + // Reconnect after the worker died; committed no-op identity survives a later root. + let retry_db = sea_orm::Database::connect(config.db_url.as_str()) + .await + .unwrap(); + let retry_mono = MonoStorage { + base: BaseStorage::new(Arc::new(retry_db)), + }; + let txn = retry_mono.get_connection().begin().await.unwrap(); + match retry_mono + .begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await + .unwrap() + { + PublicationPreparation::AlreadyCommittedNoop(receipt) => { + assert!(committed); + assert_eq!(Some(receipt.clone()), original); + assert_eq!(receipt.observed_sequence, 0); + assert_eq!(receipt.observed_root_commit, "a".repeat(40)); + assert_eq!(receipt.landed_commit_id, "tip"); + txn.commit().await.unwrap(); + } + PublicationPreparation::Prepared(_) => { + assert!(!committed); + txn.rollback().await.unwrap(); + } + PublicationPreparation::AlreadyCommitted(_) => panic!("no-op must not publish"), + } + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + if committed { "c" } else { "a" }.repeat(40) + ); + } +} diff --git a/src/jupiter/storage/push_queue_storage.rs b/src/jupiter/storage/push_queue_storage.rs index 06878580..eeae4a68 100644 --- a/src/jupiter/storage/push_queue_storage.rs +++ b/src/jupiter/storage/push_queue_storage.rs @@ -241,8 +241,9 @@ impl PushQueueStorage { }); } // operation_states race: winner already committed — classify by read. + let replay_txn = self.get_connection().begin().await?; if let Some(existing) = push_queue::Entity::find() - .filter(push_queue::Column::Kind.eq(kind)) + .filter(push_queue::Column::Kind.eq(kind.clone())) .filter(push_queue::Column::Path.eq(path)) .filter(push_queue::Column::OperationId.eq(operation_id)) .filter(push_queue::Column::Status.is_in([ @@ -250,9 +251,23 @@ impl PushQueueStorage { PushQueueStatusEnum::Running, PushQueueStatusEnum::Done, ])) - .one(self.get_connection()) + .one(&replay_txn) .await? { + if matches!(&kind, PushQueueKindEnum::Push | PushQueueKindEnum::Merge) { + let mut candidate = existing.clone(); + candidate.old_id = old_id.to_owned(); + candidate.new_id = new_id.to_owned(); + candidate.requester = requester.map(str::to_owned); + candidate.payload = payload.clone(); + crate::jupiter::storage::mono_storage::MonoStorage { + base: self.base.clone(), + } + .validate_queue_publication_replay_in_txn(&replay_txn, &candidate) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } + replay_txn.commit().await?; return match existing.status { PushQueueStatusEnum::Done => Ok(EnqueueOutcome::Replay { id: existing.id, @@ -264,6 +279,7 @@ impl PushQueueStorage { _ => unreachable!("filter restricts status"), }; } + replay_txn.rollback().await?; return Err(MegaError::Other(format!( "unique conflict without classifiable row: {msg}" ))); @@ -283,6 +299,29 @@ impl PushQueueStorage { Some(outcome) => { match &outcome { EnqueueOutcome::Adopted { .. } | EnqueueOutcome::Replay { .. } => { + if matches!(&kind, PushQueueKindEnum::Push | PushQueueKindEnum::Merge) { + let id = match &outcome { + EnqueueOutcome::Adopted { id } + | EnqueueOutcome::Replay { id, .. } => *id, + _ => unreachable!("adopt or replay"), + }; + let mut candidate = push_queue::Entity::find_by_id(id) + .one(&txn) + .await? + .ok_or_else(|| { + MegaError::Other("queue replay row missing".into()) + })?; + candidate.old_id = old_id.to_owned(); + candidate.new_id = new_id.to_owned(); + candidate.requester = requester.map(str::to_owned); + candidate.payload = payload.clone(); + crate::jupiter::storage::mono_storage::MonoStorage { + base: self.base.clone(), + } + .validate_queue_publication_replay_in_txn(&txn, &candidate) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } txn.commit().await?; } EnqueueOutcome::Rejected { .. } => { From 97a9d4e4a27ee14fbccb1deaf6a3924a9b3fc214 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 08:00:30 +0800 Subject: [PATCH 04/14] fix(mst2): fail closed for native merge writer --- src/api/router/snapshot_router.rs | 118 +- src/callisto/mod.rs | 3 + src/callisto/mst2_native_head.rs | 20 + src/callisto/mst2_native_publication.rs | 27 + src/callisto/mst2_publication.rs | 8 +- src/callisto/mst2_queue_noop_receipt.rs | 34 + src/callisto/push_queue.rs | 3 + src/ceres/snapshot/retention.rs | 447 +++++-- src/ceres/snapshot/runtime.rs | 452 ++++++- ...100_add_mst2_publication_request_digest.rs | 48 + .../m20261005_000200_add_mst2_native_head.rs | 72 ++ src/jupiter/migration/mod.rs | 4 + .../service/native_publication_push_tests.rs | 248 ++++ src/jupiter/service/push_queue_service.rs | 738 ++++++++++- src/jupiter/storage/mod.rs | 8 +- src/jupiter/storage/mono_storage.rs | 141 +-- .../storage/mst2_publication_storage.rs | 514 ++++++++ src/jupiter/storage/mst2_publication_tests.rs | 1083 +++++++++++++++++ .../storage/native_publication_storage.rs | 535 ++++++++ .../storage/native_publication_tests.rs | 378 ++++++ src/jupiter/storage/push_queue_storage.rs | 165 ++- src/jupiter/tests.rs | 6 +- 22 files changed, 4664 insertions(+), 388 deletions(-) create mode 100644 src/callisto/mst2_native_head.rs create mode 100644 src/callisto/mst2_native_publication.rs create mode 100644 src/callisto/mst2_queue_noop_receipt.rs create mode 100644 src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs create mode 100644 src/jupiter/migration/m20261005_000200_add_mst2_native_head.rs create mode 100644 src/jupiter/service/native_publication_push_tests.rs create mode 100644 src/jupiter/storage/mst2_publication_storage.rs create mode 100644 src/jupiter/storage/mst2_publication_tests.rs create mode 100644 src/jupiter/storage/native_publication_storage.rs create mode 100644 src/jupiter/storage/native_publication_tests.rs diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index 556897e3..9396caa4 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -82,6 +82,22 @@ tokio::task_local! { static REQUEST_ID: String; } +#[cfg(test)] +tokio::task_local! { + static NATIVE_RESOLVE_BARRIERS: (std::sync::Arc, std::sync::Arc); +} + +#[cfg(test)] +pub(crate) async fn with_native_resolve_barriers( + captured: std::sync::Arc, + release: std::sync::Arc, + future: F, +) -> F::Output { + NATIVE_RESOLVE_BARRIERS + .scope((captured, release), future) + .await +} + /// The request id of the in-flight request, for error envelopes. pub(crate) fn current_request_id() -> String { REQUEST_ID.try_with(|id| id.clone()).unwrap_or_default() @@ -349,21 +365,61 @@ async fn resolve(state: State, body: Bytes) -> Result, body: Bytes) -> Result, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/mst2_native_publication.rs b/src/callisto/mst2_native_publication.rs new file mode 100644 index 00000000..449be1db --- /dev/null +++ b/src/callisto/mst2_native_publication.rs @@ -0,0 +1,27 @@ +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "mst2_native_publication")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub receipt_id: i64, + pub namespace: String, + pub instance_id: String, + pub sequence: i64, + pub writer_epoch: i64, + pub old_root_commit: String, + pub old_root_tree: String, + pub root_commit: String, + pub root_tree: String, + pub origin_path: String, + pub origin_ref: String, + pub old_path_commit: Option, + pub old_path_tree: Option, + pub path_commit: String, + pub path_tree: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/mst2_publication.rs b/src/callisto/mst2_publication.rs index ff69518e..603b6509 100644 --- a/src/callisto/mst2_publication.rs +++ b/src/callisto/mst2_publication.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; pub struct Model { #[sea_orm(primary_key, auto_increment = true)] pub id: i64, - /// Deterministic writer-side operation id (e.g. push old→new). + /// Server-assigned writer identity (e.g. `mst2:trunk-queue:`). pub operation_id: String, /// Namespace path this publication advances (e.g. "/"). pub namespace: String, @@ -29,6 +29,12 @@ pub struct Model { pub writer_epoch: i64, /// `trunk_push` | `web_edit` | `import` | … (spec 09 §5 writer matrix). pub writer_kind: String, + /// Missing on legacy receipts; never reconstructed from current refs. + #[sea_orm(column_type = "Text", nullable)] + pub request_digest: Option, + pub request_digest_version: Option, + /// NULL preserves historical path-only receipts; version 1 requires its certificate. + pub native_certificate_version: Option, pub created_at: DateTimeWithTimeZone, } diff --git a/src/callisto/mst2_queue_noop_receipt.rs b/src/callisto/mst2_queue_noop_receipt.rs new file mode 100644 index 00000000..15c3b137 --- /dev/null +++ b/src/callisto/mst2_queue_noop_receipt.rs @@ -0,0 +1,34 @@ +//! Committed trunk-queue no-op results, separate from visible publications. + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_queue_noop_receipt")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true)] + pub id: i64, + #[sea_orm(column_type = "Text")] + pub operation_id: String, + #[sea_orm(column_type = "Text")] + pub namespace: String, + #[sea_orm(column_type = "Text")] + pub request_digest: String, + pub request_digest_version: i32, + pub writer_epoch: i64, + #[sea_orm(column_type = "Text")] + pub writer_kind: String, + pub observed_sequence: i64, + #[sea_orm(column_type = "Text")] + pub observed_root_commit: String, + #[sea_orm(column_type = "Text")] + pub observed_root_tree: String, + #[sea_orm(column_type = "Text")] + pub landed_commit_id: String, + pub created_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/push_queue.rs b/src/callisto/push_queue.rs index 01638727..a8dea3d6 100644 --- a/src/callisto/push_queue.rs +++ b/src/callisto/push_queue.rs @@ -42,6 +42,9 @@ pub struct Model { pub expected_commit_hash: Option, #[sea_orm(column_type = "Text", nullable)] pub expected_tree_hash: Option, + pub expected_native_sequence: Option, + pub expected_native_epoch: Option, + pub expected_native_certificate: Option, pub pending_action: Option, pub enqueued_at: DateTimeWithTimeZone, pub started_at: Option, diff --git a/src/ceres/snapshot/retention.rs b/src/ceres/snapshot/retention.rs index 54bba4da..fa9a12f6 100644 --- a/src/ceres/snapshot/retention.rs +++ b/src/ceres/snapshot/retention.rs @@ -105,7 +105,7 @@ pub enum ReapDecision { #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct CollectionReport { - /// Node ids that became unreachable this run. + /// Node ids atomically marked DELETING this run. pub unreachable: Vec, /// Bytes that would be / were reclaimed. pub reclaimed_bytes: u64, @@ -140,16 +140,30 @@ impl Reaper for NoopReaper { pub trait RetentionStore { fn node(&self, id: &str) -> Option; fn root_covers(&self, node_id: &str) -> bool; + /// Count incoming edges from parents not yet removed, including + /// DELETING parents whose physical reclaim has not succeeded. fn live_incoming(&self, node_id: &str) -> usize; - /// Idempotent: create the node if absent, upsert edges and roots in - /// one atomic step (spec §6 "no empty window"). + /// Idempotent: retain one LIVE node with its edges and root coverage. fn retain( &self, node: RetentionNode, edges: &[RetentionEdge], roots: &[RetentionRoot], + ) -> Result<(), SnapshotError> { + self.retain_group(std::slice::from_ref(&node), edges, roots) + } + /// Atomically retain the entire group, covering every supplied node + /// with `roots`. All nodes and edge endpoints must be LIVE; acquiring + /// a DELETING node is forbidden. Any error leaves the graph unchanged. + fn retain_group( + &self, + nodes: &[RetentionNode], + edges: &[RetentionEdge], + roots: &[RetentionRoot], ) -> Result<(), SnapshotError>; - /// Atomically CAS a node LIVE→DELETING; false if it is no longer LIVE. + /// Atomically check zero roots/incoming references and CAS LIVE→DELETING. + /// A parent still protects its children until the parent is removed, + /// including while the parent is DELETING and physical reclaim is pending. fn mark_deleting(&self, id: &str) -> bool; /// Remove a DELETING node after the reaper succeeds. fn remove(&self, id: &str); @@ -182,14 +196,6 @@ impl RetentionCoordinator { parents: &[String], roots: &[RetentionRoot], ) -> Result<(), SnapshotError> { - for p in parents { - if self.store.node(p).is_none() { - return Err(SnapshotError::new( - SnapshotErrorCode::Internal, - format!("retention edge to unknown parent {p}"), - )); - } - } let edges: Vec = parents .iter() .map(|p| RetentionEdge { @@ -200,38 +206,24 @@ impl RetentionCoordinator { self.store.retain(node, &edges, roots) } - /// Add coverage from a root to a node, atomically re-lifting it out of - /// a DELETING state (spec §10). The root is also materialized as a live - /// anchor node so reachability can start from it. The store's `retain` - /// is the upsert, so both the fresh and the already-known cases take - /// the same path. + /// Acquire root coverage for the complete group in one store operation. + /// The root is also materialized as a LIVE anchor. If any covered node + /// is DELETING, neither the anchor nor any partial coverage is retained. pub fn pin_root( &self, root: &RetentionRoot, covered: &[RetentionNode], ) -> Result<(), SnapshotError> { - let roots = std::slice::from_ref(root); - self.store.retain( - RetentionNode { - id: root.key(), - kind: RetainedKind::Frame, - state: NodeState::Live, - bytes: 0, - }, - &[], - roots, - )?; - for n in covered { - self.store.retain( - RetentionNode { - state: NodeState::Live, - ..n.clone() - }, - &[], - roots, - )?; - } - Ok(()) + let mut nodes = Vec::with_capacity(covered.len() + 1); + nodes.push(RetentionNode { + id: root.key(), + kind: RetainedKind::Frame, + state: NodeState::Live, + bytes: 0, + }); + nodes.extend_from_slice(covered); + self.store + .retain_group(&nodes, &[], std::slice::from_ref(root)) } pub fn release(&self, root: &RetentionRoot) { @@ -239,11 +231,10 @@ impl RetentionCoordinator { } /// One collection pass (spec 10 §6 steps 1–7): - /// mark unreachable LIVE nodes DELETING, then reap. Nodes marked - /// DELETING are skipped entirely on later passes if a new root/edge - /// raced in only after the CAS — here reachability is recomputed under - /// the store lock, so a retain concurrent with a pass either observes - /// the node (keeping it LIVE) or lands before the next pass. + /// mark unreachable LIVE nodes DELETING, then reap. The reachability + /// scan selects candidates; the store rechecks roots and incoming + /// references atomically with the CAS. Acquisition either wins before + /// that CAS or fails because the node is already DELETING. pub fn collect(&self, reaper: &R) -> Result { let live = self.store.all_live(); let reachable = self.reachable_set(); @@ -254,18 +245,11 @@ impl RetentionCoordinator { continue; } if !self.store.mark_deleting(&node.id) { - // Lost the CAS: a new reference made it LIVE again. + // Another collector or a newly acquired reference won. continue; } report.unreachable.push(node.id.clone()); report.reclaimed_bytes += node.bytes; - // Re-verify after the CAS: the store serializes retain against - // mark, so a zero incoming count here is stable for this pass. - if self.store.root_covers(&node.id) || self.store.live_incoming(&node.id) > 0 { - // A concurrent retain resurrected it; the CAS semantics of - // the store keep it LIVE, so do not reap. - continue; - } if reaper.physical() { reaper.reap(&node)?; self.store.remove(&node.id); @@ -311,6 +295,8 @@ pub mod mem { edges: HashSet, /// node id -> set of root keys covering it. roots: HashMap>, + #[cfg(test)] + fail_next_retain: bool, } /// Single-Mutex store; the lock is the serialization point that makes @@ -327,6 +313,14 @@ pub mod mem { } } + #[cfg(test)] + impl InMemoryRetentionStore { + /// Inject a backend failure after validation and before committing. + pub(crate) fn fail_next_retain_for_test(&self) { + self.inner.lock().unwrap().fail_next_retain = true; + } + } + impl RetentionStore for InMemoryRetentionStore { fn node(&self, id: &str) -> Option { self.inner.lock().unwrap().nodes.get(id).cloned() @@ -350,41 +344,82 @@ pub mod mem { .count() } - fn retain( + fn retain_group( &self, - node: RetentionNode, + nodes: &[RetentionNode], edges: &[RetentionEdge], roots: &[RetentionRoot], ) -> Result<(), SnapshotError> { let mut g = self.inner.lock().unwrap(); - // Atomic re-lift: never downgrade a LIVE node to DELETING. - g.nodes - .entry(node.id.clone()) - .and_modify(|existing| { - existing.bytes = node.bytes; - existing.kind = node.kind; - existing.state = NodeState::Live; - }) - .or_insert_with(|| node.clone()); - // Defensive: do not insert an edge to a missing child node. - for e in edges { - if !g.nodes.contains_key(&e.child) || !g.nodes.contains_key(&e.parent) { + // Validate the entire transaction before mutating any graph data. + let mut staged = HashMap::new(); + for node in nodes { + if node.state != NodeState::Live + || g.nodes + .get(&node.id) + .is_some_and(|existing| existing.state != NodeState::Live) + { + return Err(SnapshotError::new( + SnapshotErrorCode::ObjectUnavailable, + format!("retention acquire requires LIVE node {}", node.id), + )); + } + if staged + .insert(node.id.as_str(), node) + .is_some_and(|previous| previous != node) + { return Err(SnapshotError::new( SnapshotErrorCode::Internal, - "retention edge references missing node", + "retention group has conflicting node definitions", )); } - g.edges.insert(e.clone()); } - let entry = g.roots.entry(node.id.clone()).or_default(); - for r in roots { - entry.insert(r.key()); + for e in edges { + for id in [&e.parent, &e.child] { + match staged.get(id.as_str()).copied().or_else(|| g.nodes.get(id)) { + Some(node) if node.state == NodeState::Live => {} + Some(_) => { + return Err(SnapshotError::new( + SnapshotErrorCode::ObjectUnavailable, + format!("retention edge requires LIVE endpoint {id}"), + )); + } + None => { + return Err(SnapshotError::new( + SnapshotErrorCode::Internal, + format!("retention edge references missing node {id}"), + )); + } + } + } + } + #[cfg(test)] + if std::mem::take(&mut g.fail_next_retain) { + return Err(SnapshotError::new( + SnapshotErrorCode::Internal, + "injected retention commit failure", + )); + } + for node in nodes { + g.nodes.insert(node.id.clone(), node.clone()); + let entry = g.roots.entry(node.id.clone()).or_default(); + for root in roots { + entry.insert(root.key()); + } } + g.edges.extend(edges.iter().cloned()); Ok(()) } fn mark_deleting(&self, id: &str) -> bool { let mut g = self.inner.lock().unwrap(); + if g.roots.get(id).is_some_and(|roots| !roots.is_empty()) + || g.edges + .iter() + .any(|edge| edge.child == id && g.nodes.contains_key(&edge.parent)) + { + return false; + } match g.nodes.get_mut(id) { Some(n) if n.state == NodeState::Live => { n.state = NodeState::Deleting; @@ -396,6 +431,13 @@ pub mod mem { fn remove(&self, id: &str) { let mut g = self.inner.lock().unwrap(); + if !g + .nodes + .get(id) + .is_some_and(|node| node.state == NodeState::Deleting) + { + return; + } g.nodes.remove(id); g.edges.retain(|e| e.parent != id && e.child != id); g.roots.remove(id); @@ -435,6 +477,15 @@ pub mod mem { #[cfg(test)] mod tests { + use std::{ + sync::{ + Arc, + mpsc::{Receiver, SyncSender, sync_channel}, + }, + thread, + time::Duration, + }; + use super::{mem::InMemoryRetentionStore, *}; fn node(id: &str, bytes: u64) -> RetentionNode { @@ -450,6 +501,243 @@ mod tests { RetentionCoordinator::new(InMemoryRetentionStore::default()) } + /// Pause a collection after its scan, immediately before the store CAS. + /// Timeouts make both sides of the forced interleaving bounded. + struct PausedMarkStore { + inner: InMemoryRetentionStore, + ready: SyncSender<()>, + resume: Mutex>, + } + + impl RetentionStore for PausedMarkStore { + fn node(&self, id: &str) -> Option { + self.inner.node(id) + } + + fn root_covers(&self, node_id: &str) -> bool { + self.inner.root_covers(node_id) + } + + fn live_incoming(&self, node_id: &str) -> usize { + self.inner.live_incoming(node_id) + } + + fn retain_group( + &self, + nodes: &[RetentionNode], + edges: &[RetentionEdge], + roots: &[RetentionRoot], + ) -> Result<(), SnapshotError> { + self.inner.retain_group(nodes, edges, roots) + } + + fn mark_deleting(&self, id: &str) -> bool { + self.ready.send(()).unwrap(); + self.resume + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .expect("test did not resume deletion"); + self.inner.mark_deleting(id) + } + + fn remove(&self, id: &str) { + self.inner.remove(id); + } + + fn release_root(&self, root: &RetentionRoot) { + self.inner.release_root(root); + } + + fn all_live(&self) -> Vec { + self.inner.all_live() + } + + fn children(&self, parent: &str) -> Vec { + self.inner.children(parent) + } + } + + #[test] + fn failed_pin_group_leaves_no_anchor_or_partial_coverage() { + let c = coord(); + let original = node("existing", 10); + c.store().retain(original.clone(), &[], &[]).unwrap(); + c.store().retain(node("deleting", 7), &[], &[]).unwrap(); + assert!(c.store().mark_deleting("deleting")); + let root = RetentionRoot::Lease("new".into()); + + // The last node fails after earlier entries would have been written + // by the old per-node pin loop, including an update to existing. + let err = c + .pin_root( + &root, + &[node("fresh", 1), node("existing", 99), node("deleting", 7)], + ) + .unwrap_err(); + assert_eq!(err.code, SnapshotErrorCode::ObjectUnavailable); + assert!(c.store().node(&root.key()).is_none()); + assert!(c.store().node("fresh").is_none()); + assert_eq!(c.store().node("existing"), Some(original)); + assert!(!c.store().root_covers("existing")); + assert!(!c.store().root_covers("deleting")); + assert_eq!( + c.store().node("deleting").unwrap().state, + NodeState::Deleting + ); + } + + #[test] + fn supplied_deleting_node_cannot_be_created_or_pinned() { + let c = coord(); + let root = RetentionRoot::Pin("invalid".into()); + let mut deleting = node("absent", 1); + deleting.state = NodeState::Deleting; + assert_eq!( + c.pin_root(&root, &[deleting]).unwrap_err().code, + SnapshotErrorCode::ObjectUnavailable + ); + assert!(c.store().all_live().is_empty()); + assert!(c.store().node("absent").is_none()); + assert!(c.store().node(&root.key()).is_none()); + } + + #[test] + fn late_invalid_edge_rolls_back_nodes_edges_and_roots() { + let c = coord(); + c.store().retain(node("parent", 0), &[], &[]).unwrap(); + let root = RetentionRoot::Lease("new".into()); + let edges = [ + RetentionEdge { + parent: "parent".into(), + child: "fresh".into(), + }, + RetentionEdge { + parent: "missing".into(), + child: "fresh".into(), + }, + ]; + let err = c + .store() + .retain(node("fresh", 2), &edges, &[root]) + .unwrap_err(); + assert_eq!(err.code, SnapshotErrorCode::Internal); + assert!(c.store().node("fresh").is_none()); + assert!(c.store().children("parent").is_empty()); + assert!(!c.store().root_covers("fresh")); + assert_eq!(c.store().live_incoming("fresh"), 0); + } + + #[test] + fn injected_commit_failure_preserves_existing_root_and_can_retry() { + let c = coord(); + let old = RetentionRoot::Pin("old".into()); + let new = RetentionRoot::Lease("new".into()); + c.store() + .retain(node("existing", 1), &[], std::slice::from_ref(&old)) + .unwrap(); + c.store().fail_next_retain_for_test(); + assert_eq!( + c.pin_root(&new, &[node("existing", 1), node("fresh", 2)]) + .unwrap_err() + .code, + SnapshotErrorCode::Internal + ); + assert!(c.store().node(&new.key()).is_none()); + assert!(c.store().node("fresh").is_none()); + assert!(c.store().root_covers("existing")); + c.release(&old); + assert!( + !c.store().root_covers("existing"), + "failed pin added no new root" + ); + + c.pin_root(&new, &[node("existing", 1), node("fresh", 2)]) + .unwrap(); + assert!(c.is_retained("existing")); + assert!(c.is_retained("fresh")); + } + + #[test] + fn collection_rechecks_roots_and_edges_acquired_after_its_scan() { + for via_edge in [false, true] { + let (ready_tx, ready_rx) = sync_channel(1); + let (resume_tx, resume_rx) = sync_channel(1); + let c = Arc::new(RetentionCoordinator::new(PausedMarkStore { + inner: InMemoryRetentionStore::default(), + ready: ready_tx, + resume: Mutex::new(resume_rx), + })); + c.store().retain(node("candidate", 1), &[], &[]).unwrap(); + let root = RetentionRoot::Lease("late".into()); + if via_edge { + c.store() + .retain(node("parent", 0), &[], std::slice::from_ref(&root)) + .unwrap(); + } + let collector = { + let c = Arc::clone(&c); + thread::spawn(move || c.collect(&NoopReaper)) + }; + ready_rx + .recv_timeout(Duration::from_secs(5)) + .expect("collector did not reach its deletion CAS"); + // Both references land after candidate selection, before CAS. + if via_edge { + c.retain(node("candidate", 1), &["parent".into()], &[]) + .unwrap(); + } else { + c.pin_root(&root, &[node("candidate", 1)]).unwrap(); + } + resume_tx.send(()).unwrap(); + let report = collector.join().unwrap().unwrap(); + assert!(report.unreachable.is_empty()); + assert_eq!(c.store().node("candidate").unwrap().state, NodeState::Live); + assert!(c.is_retained("candidate")); + } + } + + #[test] + fn deleting_parent_rejects_new_edges_and_protects_existing_children_until_removed() { + let c = coord(); + c.store().retain(node("parent", 0), &[], &[]).unwrap(); + c.retain(node("child", 1), &["parent".into()], &[]).unwrap(); + assert!(c.store().mark_deleting("parent")); + assert!(!c.store().mark_deleting("child")); + assert_eq!(c.store().live_incoming("child"), 1); + assert_eq!( + c.retain(node("new-child", 2), &["parent".into()], &[]) + .unwrap_err() + .code, + SnapshotErrorCode::ObjectUnavailable + ); + assert!(c.store().node("new-child").is_none()); + c.store().remove("parent"); + assert_eq!(c.store().live_incoming("child"), 0); + assert!(c.store().mark_deleting("child")); + } + + #[test] + fn retain_cannot_add_an_edge_to_a_deleting_child() { + let c = coord(); + c.store().retain(node("child", 1), &[], &[]).unwrap(); + assert!(c.store().mark_deleting("child")); + let edge = RetentionEdge { + parent: "new-parent".into(), + child: "child".into(), + }; + assert_eq!( + c.store() + .retain(node("new-parent", 0), &[edge], &[]) + .unwrap_err() + .code, + SnapshotErrorCode::ObjectUnavailable + ); + assert!(c.store().node("new-parent").is_none()); + assert_eq!(c.store().node("child").unwrap().state, NodeState::Deleting); + assert_eq!(c.store().live_incoming("child"), 0); + } + #[test] fn leaf_dies_when_root_released_but_other_root_keeps_it() { // root A and B both cover a shared leaf; releasing A must not @@ -486,8 +774,8 @@ mod tests { let r = c.collect(&NoopReaper).unwrap(); assert_eq!(r.unreachable, vec!["orphan".to_string()]); assert_eq!(r.reclaimed_bytes, 9); - // Releasing the root cascades unreachability to a and b only after - // the anchor itself is collected (the anchor node is root-covered). + // Releasing the root makes the chain unreachable, but children stay + // LIVE while their parents await physical removal under NoopReaper. c.release(&root); let _ = c.collect(&NoopReaper).unwrap(); assert!(!c.is_retained("b")); @@ -521,7 +809,7 @@ mod tests { ) .unwrap(); assert_eq!(c.store().live_incoming("shared"), 2); - // Remove p1's coverage and p1 itself; shared still has p2. + // Drop p1's coverage; its pending deletion must not affect p2's view. c.release(&RetentionRoot::Pin("r1".into())); c.collect(&NoopReaper).unwrap(); assert!(c.store().node("shared").is_some()); @@ -562,12 +850,12 @@ mod tests { let c = coord(); c.store().retain(node("x", 3), &[], &[]).unwrap(); assert!(c.collect(&Fail).is_err()); - // Node remains (DELETING) so a retry can replay; it is not lost. - assert!(c.store().node("x").is_some()); + // Node remains DELETING; physical retry/recovery is a later slice. + assert_eq!(c.store().node("x").unwrap().state, NodeState::Deleting); } #[test] - fn new_root_revives_a_node_before_it_is_collected() { + fn new_root_protects_a_live_node_before_it_is_collected() { let c = coord(); c.store().retain(node("x", 4), &[], &[]).unwrap(); // Before any collect, a lease covers it. @@ -644,7 +932,12 @@ mod tests { c.retain(node("ch", 2), &["p".to_string()], &[]).unwrap(); c.release(&RetentionRoot::Lease("l".into())); c.collect(&Delete).unwrap(); + // A child selected before its parent was removed waits for the + // next pass; iteration order must not affect the final result. + c.collect(&Delete).unwrap(); // Both nodes and their edge are gone; no stale incoming edge. + assert!(c.store().node("p").is_none()); + assert!(c.store().node("ch").is_none()); assert_eq!(c.store().live_incoming("ch"), 0); } } diff --git a/src/ceres/snapshot/runtime.rs b/src/ceres/snapshot/runtime.rs index 770c4725..b8b4bce3 100644 --- a/src/ceres/snapshot/runtime.rs +++ b/src/ceres/snapshot/runtime.rs @@ -11,7 +11,7 @@ use std::{ collections::HashMap, - sync::{Mutex, OnceLock}, + sync::{Mutex, MutexGuard, OnceLock}, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -101,16 +101,16 @@ impl Mst2Runtime { st.1 } - /// Insert a context and create its first lease. The lease also becomes - /// a retention root covering the snapshot's derived metadata root - /// (spec 10 §5: active leases pin what a fixed view needs). + /// Acquire the complete in-memory retention group before exposing a + /// context or lease. These placeholder anchors do not yet represent + /// the complete durable page/chunk graph required by T06. pub fn insert_context( &self, built: BuiltDescriptor, commit_oid: &str, root_tree_oid: &str, lease_seconds: u64, - ) -> SnapshotContext { + ) -> Result { let lease_id = Uuid::new_v4().to_string(); let expires = now_unix() + lease_seconds.clamp(1, 3600); let snapshot_id = built.snapshot_id.clone(); @@ -122,23 +122,9 @@ impl Mst2Runtime { lease_expires_at_unix: expires, authorization_epoch: 1, }; - self.contexts.lock().unwrap().insert( - snapshot_id.clone(), - ContextData { - built: built.clone(), - commit_oid: commit_oid.to_string(), - root_tree_oid: root_tree_oid.to_string(), - }, - ); - self.leases.lock().unwrap().insert( - lease_id.clone(), - LeaseRec { - snapshot_id: snapshot_id.clone(), - expires_at_unix: expires, - }, - ); - // Retention root for the lease: the metadata-root page node and the - // snapshot's chunk projections stay reachable while it is active. + // Serialize registration with release/expiry/renewal. No path holds + // contexts while acquiring leases, so this lock order cannot cycle. + let mut leases = self.leases.lock().unwrap(); let root = crate::ceres::snapshot::retention::RetentionRoot::Lease(lease_id.clone()); let page_node = crate::ceres::snapshot::retention::RetentionNode { id: format!("page:{}", built.metadata_root), @@ -152,15 +138,24 @@ impl Mst2Runtime { state: crate::ceres::snapshot::retention::NodeState::Live, bytes: 0, }; - if let Err(e) = self - .retention - .pin_root(&root, &[page_node, projection_node]) - { - // Coverage failure is surfaced, not swallowed: a lease that - // cannot pin its view must not pretend to. - tracing::warn!(error = %e, "retention pin for new lease failed"); - } - ctx + self.retention + .pin_root(&root, &[page_node, projection_node])?; + self.contexts.lock().unwrap().insert( + snapshot_id.clone(), + ContextData { + built, + commit_oid: commit_oid.to_string(), + root_tree_oid: root_tree_oid.to_string(), + }, + ); + leases.insert( + lease_id, + LeaseRec { + snapshot_id, + expires_at_unix: expires, + }, + ); + Ok(ctx) } /// Look up a snapshot context by snapshot_id. It is servable while at @@ -194,7 +189,7 @@ impl Mst2Runtime { /// as it goes. fn active_lease(&self, snapshot_id: &str) -> Result<(String, u64), SnapshotError> { let mut leases = self.leases.lock().unwrap(); - leases.retain(|_, r| r.expires_at_unix >= now_unix()); + self.prune_expired_leases(&mut leases, now_unix()); leases .iter() .find(|(_, r)| r.snapshot_id == snapshot_id) @@ -207,6 +202,22 @@ impl Mst2Runtime { }) } + /// The leases lock serializes expiry with renewal and registration; + /// releasing one expired lease never removes another lease's coverage. + /// The current in-memory store's release operation is infallible. + fn prune_expired_leases(&self, leases: &mut HashMap, now: u64) { + leases.retain(|lease_id, rec| { + if rec.expires_at_unix >= now { + return true; + } + self.retention + .release(&crate::ceres::snapshot::retention::RetentionRoot::Lease( + lease_id.clone(), + )); + false + }); + } + /// Validate the lease a request presented (spec 04 §1: identity lives in /// the `X-Mega-Snapshot-Lease` header, and a read path must check it — /// knowing the snapshot id alone is not a capability). Any lease that is @@ -216,8 +227,8 @@ impl Mst2Runtime { /// existed is answered identically: probing with lease ids must not /// distinguish released from forged. /// - /// Expired-row pruning happens in `active_lease`, not here: this runs on - /// every authenticated read and must stay a single key lookup. + /// Expired-row pruning happens in `active_lease` and collection: this + /// runs on every authenticated read and stays a single key lookup. pub fn validate_lease(&self, snapshot_id: &str, lease_id: &str) -> Result<(), SnapshotError> { let leases = self.leases.lock().unwrap(); match leases.get(lease_id) { @@ -237,8 +248,24 @@ impl Mst2Runtime { lease_id: &str, lease_seconds: u64, ) -> Result { - let now = now_unix(); - let mut leases = self.leases.lock().unwrap(); + Self::renew_lease_with_clock( + lease_id, + lease_seconds, + || self.leases.lock().unwrap(), + now_unix, + ) + } + + /// Sample the clock only after acquiring the lease lock. Separate + /// sources let tests force a deadline crossing during lock contention. + fn renew_lease_with_clock<'a>( + lease_id: &str, + lease_seconds: u64, + acquire_leases: impl FnOnce() -> MutexGuard<'a, HashMap>, + clock: impl FnOnce() -> u64, + ) -> Result { + let mut leases = acquire_leases(); + let now = clock(); let Some(rec) = leases.get_mut(lease_id) else { return Err(SnapshotError::new( SnapshotErrorCode::LeaseUnknown, @@ -268,7 +295,8 @@ impl Mst2Runtime { /// only drops the retention root so a later collection pass may /// reclaim derived pages no other lease/pin covers. pub fn release_lease(&self, lease_id: &str) -> bool { - let removed = self.leases.lock().unwrap().remove(lease_id).is_some(); + let mut leases = self.leases.lock().unwrap(); + let removed = leases.remove(lease_id).is_some(); self.retention .release(&crate::ceres::snapshot::retention::RetentionRoot::Lease( lease_id.to_string(), @@ -282,6 +310,7 @@ impl Mst2Runtime { pub fn collect_retention( &self, ) -> Result { + self.prune_expired_leases(&mut self.leases.lock().unwrap(), now_unix()); self.retention .collect(&crate::ceres::snapshot::retention::NoopReaper) } @@ -336,20 +365,24 @@ pub fn rfc3339(unix: u64) -> String { #[cfg(test)] mod tests { - use super::*; + use std::{ + sync::{ + Arc, TryLockError, + atomic::{AtomicU64, Ordering}, + mpsc::sync_channel, + }, + thread, + }; - #[test] - fn rfc3339_known_values() { - assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z"); - assert_eq!(rfc3339(1_789_525_722), "2026-09-16T02:28:42Z"); - } + use mst2_codec::descriptor::ServingDescriptor; - #[tokio::test] - async fn lease_lifecycle_drives_retention() { - use crate::ceres::snapshot::retention::{ - NodeState, RetainedKind, RetentionRoot, RetentionStore as _, - }; - let r = Mst2Runtime { + use super::*; + use crate::ceres::snapshot::retention::{ + NodeState, RetainedKind, RetentionRoot, RetentionStore as _, + }; + + fn isolated_runtime() -> Mst2Runtime { + Mst2Runtime { hmac_key: blake3_key(), contexts: Mutex::new(HashMap::new()), leases: Mutex::new(HashMap::new()), @@ -357,9 +390,324 @@ mod tests { crate::ceres::snapshot::retention::mem::InMemoryRetentionStore::default(), ), tip_state: Mutex::new((String::new(), 0u64)), + } + } + + fn descriptor(root_byte: u8) -> BuiltDescriptor { + let descriptor = ServingDescriptor { + instance_uuid: [0x11; 16], + namespace_view_id: [0x22; 32], + scope: "/".into(), + metadata_root: [root_byte; 32], + }; + BuiltDescriptor { + instance_id: Uuid::from_bytes(descriptor.instance_uuid).to_string(), + snapshot_id: format!( + "sha256:{}", + crate::ceres::snapshot::view::hex(&descriptor.snapshot_id().unwrap()) + ), + metadata_root: format!( + "sha256:{}", + crate::ceres::snapshot::view::hex(&descriptor.metadata_root) + ), + descriptor, + } + } + + #[test] + fn pin_failure_exposes_no_context_lease_or_partial_graph_and_can_retry() { + let r = isolated_runtime(); + let built = descriptor(3); + r.retention.store().fail_next_retain_for_test(); + assert_eq!( + r.insert_context(built.clone(), "commit", "tree", 60) + .unwrap_err() + .code, + SnapshotErrorCode::Internal + ); + assert!(r.contexts.lock().unwrap().is_empty()); + assert!(r.leases.lock().unwrap().is_empty()); + assert!(r.retention.store().all_live().is_empty()); + assert_eq!( + r.context(&built.snapshot_id).unwrap_err().code, + SnapshotErrorCode::SnapshotGone + ); + + let ctx = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + r.validate_lease(&built.snapshot_id, &ctx.lease_id).unwrap(); + assert!( + r.retention + .is_retained(&format!("page:{}", built.metadata_root)) + ); + assert!(r.retention.is_retained("projection:commit")); + } + + #[test] + fn failed_resolve_preserves_an_existing_active_snapshot() { + let r = isolated_runtime(); + let old = descriptor(3); + let active = r + .insert_context(old.clone(), "old-commit", "old-tree", 60) + .unwrap(); + let new = descriptor(4); + r.retention.store().fail_next_retain_for_test(); + assert!( + r.insert_context(new.clone(), "new-commit", "new-tree", 60) + .is_err() + ); + assert_eq!(r.contexts.lock().unwrap().len(), 1); + assert_eq!(r.leases.lock().unwrap().len(), 1); + assert_eq!( + r.context(&old.snapshot_id).unwrap().lease_id, + active.lease_id + ); + assert_eq!( + r.context(&new.snapshot_id).unwrap_err().code, + SnapshotErrorCode::SnapshotGone + ); + assert!( + r.retention + .store() + .node(&format!("page:{}", new.metadata_root)) + .is_none() + ); + assert!(r.retention.store().node("projection:new-commit").is_none()); + } + + #[test] + fn resolve_cannot_create_a_lease_for_a_deleting_group() { + let r = isolated_runtime(); + let built = descriptor(3); + let ctx = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + assert!(r.release_lease(&ctx.lease_id)); + assert!(r.collect_retention().unwrap().reaped.is_empty()); + assert_eq!( + r.insert_context(built.clone(), "commit", "tree", 60) + .unwrap_err() + .code, + SnapshotErrorCode::ObjectUnavailable + ); + assert!(r.leases.lock().unwrap().is_empty()); + assert!( + r.retention.store().all_live().is_empty(), + "no new lease anchor leaked" + ); + assert_eq!( + r.context(&built.snapshot_id).unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + r.retention + .store() + .node(&format!("page:{}", built.metadata_root)) + .unwrap() + .state, + NodeState::Deleting + ); + } + + #[test] + fn context_expiry_releases_only_the_expired_lease_on_a_shared_snapshot() { + let r = isolated_runtime(); + let built = descriptor(3); + let expired = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + let active = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + r.leases + .lock() + .unwrap() + .get_mut(&expired.lease_id) + .unwrap() + .expires_at_unix = 0; + + assert_eq!( + r.context(&built.snapshot_id).unwrap().lease_id, + active.lease_id + ); + assert!(!r.leases.lock().unwrap().contains_key(&expired.lease_id)); + assert!( + !r.retention + .store() + .root_covers(&format!("lease:{}", expired.lease_id)) + ); + assert!( + r.retention + .store() + .root_covers(&format!("lease:{}", active.lease_id)) + ); + assert!( + r.retention + .is_retained(&format!("page:{}", built.metadata_root)) + ); + r.validate_lease(&built.snapshot_id, &active.lease_id) + .unwrap(); + assert_eq!( + r.validate_lease(&built.snapshot_id, &expired.lease_id) + .unwrap_err() + .code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + r.collect_retention().unwrap().unreachable, + vec![format!("lease:{}", expired.lease_id)] + ); + assert_eq!( + r.context(&built.snapshot_id).unwrap().lease_id, + active.lease_id + ); + + assert!(r.release_lease(&active.lease_id)); + assert!( + !r.retention + .store() + .root_covers(&format!("page:{}", built.metadata_root)) + ); + assert!(!r.retention.store().root_covers("projection:commit")); + } + + #[test] + fn collection_prunes_expired_roots_without_a_context_read() { + let r = isolated_runtime(); + let built = descriptor(3); + let ctx = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + r.leases + .lock() + .unwrap() + .get_mut(&ctx.lease_id) + .unwrap() + .expires_at_unix = 0; + let report = r.collect_retention().unwrap(); + assert_eq!(report.unreachable.len(), 3, "anchor, page and projection"); + assert!(report.reaped.is_empty(), "NoopReaper stays fail-closed"); + assert!(r.leases.lock().unwrap().is_empty()); + for id in [ + format!("lease:{}", ctx.lease_id), + format!("page:{}", built.metadata_root), + "projection:commit".into(), + ] { + assert!(report.unreachable.contains(&id)); + assert!(!r.retention.store().root_covers(&id)); + assert_eq!( + r.retention.store().node(&id).unwrap().state, + NodeState::Deleting + ); + } + } + + #[test] + fn renewal_winning_before_expiry_keeps_its_root_and_expiry_winning_prevents_renewal() { + let r = isolated_runtime(); + let built = descriptor(3); + let ctx = r + .insert_context(built.clone(), "commit", "tree", 60) + .unwrap(); + let renewed = r.renew_lease(&ctx.lease_id, 60).unwrap(); + { + let mut leases = r.leases.lock().unwrap(); + // Advance past the old deadline, before the renewed deadline. + r.prune_expired_leases(&mut leases, ctx.lease_expires_at_unix + 1); + } + r.validate_lease(&built.snapshot_id, &ctx.lease_id).unwrap(); + assert!( + r.retention + .store() + .root_covers(&format!("lease:{}", ctx.lease_id)) + ); + { + let mut leases = r.leases.lock().unwrap(); + r.prune_expired_leases(&mut leases, renewed.expires_at_unix + 1); + } + assert!( + !r.retention + .store() + .root_covers(&format!("lease:{}", ctx.lease_id)) + ); + assert_eq!( + r.renew_lease(&ctx.lease_id, 60).unwrap_err().code, + SnapshotErrorCode::LeaseUnknown + ); + assert_eq!( + r.context(&built.snapshot_id).unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + } + + #[test] + fn renewal_waiting_for_the_lease_lock_cannot_use_a_pre_expiry_clock_sample() { + let r = Arc::new(isolated_runtime()); + let ctx = r + .insert_context(descriptor(3), "commit", "tree", 60) + .unwrap(); + let clock = Arc::new(AtomicU64::new(9)); + let (blocked_tx, blocked_rx) = sync_channel(1); + let mut leases = r.leases.lock().unwrap(); + leases.get_mut(&ctx.lease_id).unwrap().expires_at_unix = 10; + let renewal = { + let r = Arc::clone(&r); + let clock = Arc::clone(&clock); + let lease_id = ctx.lease_id.clone(); + thread::spawn(move || { + Mst2Runtime::renew_lease_with_clock( + &lease_id, + 60, + || { + // Prove contention before allowing the clock to cross + // the deadline. A pre-lock sample would observe 9. + assert!(matches!(r.leases.try_lock(), Err(TryLockError::WouldBlock))); + blocked_tx.send(()).unwrap(); + r.leases.lock().unwrap() + }, + || clock.load(Ordering::SeqCst), + ) + }) }; - // Retain the snapshot's derived nodes directly, covered by the lease - // root that insert_context created. + blocked_rx + .recv_timeout(Duration::from_secs(5)) + .expect("renewal did not contend for the lease lock"); + clock.store(11, Ordering::SeqCst); + drop(leases); + + assert_eq!( + renewal.join().unwrap().unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + r.leases + .lock() + .unwrap() + .get(&ctx.lease_id) + .unwrap() + .expires_at_unix, + 10 + ); + assert_eq!(r.leases.lock().unwrap().len(), 1, "no replacement lease"); + assert!(r.collect_retention().unwrap().reaped.is_empty()); + assert!( + !r.retention + .store() + .root_covers(&format!("lease:{}", ctx.lease_id)) + ); + } + + #[test] + fn rfc3339_known_values() { + assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z"); + assert_eq!(rfc3339(1_789_525_722), "2026-09-16T02:28:42Z"); + } + + #[test] + fn lease_lifecycle_drives_retention() { + let r = isolated_runtime(); + // Cover a derived node directly with a manual test lease root. let lease_root = RetentionRoot::Lease("gc-lease-1".to_string()); let page = crate::ceres::snapshot::retention::RetentionNode { id: "page:sha256:test".to_string(), @@ -382,7 +730,7 @@ mod tests { assert_eq!(report.unreachable, vec!["page:sha256:test".to_string()]); assert_eq!(report.reclaimed_bytes, 100); assert!(report.reaped.is_empty(), "fail-closed reaper"); - // The node is DELETING, not gone: a re-resolve re-lifts it. + // The node stays DELETING; a new acquire must not resurrect it. assert_eq!( r.retention.store().node("page:sha256:test").unwrap().state, NodeState::Deleting diff --git a/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs b/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs new file mode 100644 index 00000000..8186b7c4 --- /dev/null +++ b/src/jupiter/migration/m20261005_000100_add_mst2_publication_request_digest.rs @@ -0,0 +1,48 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared( + r#"ALTER TABLE mst2_publication + ADD COLUMN IF NOT EXISTS request_digest text, + ADD COLUMN IF NOT EXISTS request_digest_version integer; + DO $$ BEGIN + ALTER TABLE mst2_publication + ADD CONSTRAINT mst2_publication_request_digest_valid CHECK ( + (request_digest IS NULL AND request_digest_version IS NULL) OR + (request_digest IS NOT NULL AND request_digest_version IS NOT NULL + AND request_digest_version > 0 + AND request_digest ~ '^sha256:[0-9a-f]{64}$') + ); + EXCEPTION WHEN duplicate_object THEN NULL; + END $$; + CREATE TABLE IF NOT EXISTS mst2_queue_noop_receipt ( + id bigserial PRIMARY KEY, + operation_id text NOT NULL UNIQUE, + namespace text NOT NULL, + request_digest text NOT NULL CHECK (request_digest ~ '^sha256:[0-9a-f]{64}$'), + request_digest_version integer NOT NULL CHECK (request_digest_version > 0), + writer_epoch bigint NOT NULL, + writer_kind text NOT NULL, + observed_sequence bigint NOT NULL CHECK (observed_sequence >= 0), + observed_root_commit text NOT NULL, + observed_root_tree text NOT NULL, + landed_commit_id text NOT NULL, + created_at timestamptz NOT NULL + );"#, + ) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Receipts remain immutable, including the absence of legacy digests. + Ok(()) + } +} diff --git a/src/jupiter/migration/m20261005_000200_add_mst2_native_head.rs b/src/jupiter/migration/m20261005_000200_add_mst2_native_head.rs new file mode 100644 index 00000000..e29118cf --- /dev/null +++ b/src/jupiter/migration/m20261005_000200_add_mst2_native_head.rs @@ -0,0 +1,72 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .get_connection() + .execute_unprepared( + r#" + ALTER TABLE mst2_publication + ADD COLUMN IF NOT EXISTS native_certificate_version integer; + ALTER TABLE push_queue + ADD COLUMN IF NOT EXISTS expected_native_sequence bigint, + ADD COLUMN IF NOT EXISTS expected_native_epoch bigint, + ADD COLUMN IF NOT EXISTS expected_native_certificate bigint; + CREATE TABLE IF NOT EXISTS mst2_native_publication ( + receipt_id bigint PRIMARY KEY REFERENCES mst2_publication(id), + namespace text NOT NULL CHECK (namespace = '/'), + instance_id text NOT NULL, + sequence bigint NOT NULL CHECK (sequence > 0), + writer_epoch bigint NOT NULL CHECK (writer_epoch > 0), + old_root_commit text NOT NULL, + old_root_tree text NOT NULL, + root_commit text NOT NULL, + root_tree text NOT NULL, + origin_path text NOT NULL, + origin_ref text NOT NULL, + old_path_commit text, + old_path_tree text, + path_commit text NOT NULL, + path_tree text NOT NULL, + CHECK ((old_path_commit IS NULL) = (old_path_tree IS NULL)), + CHECK (old_path_commit IS DISTINCT FROM path_commit), + UNIQUE (namespace, sequence) + ); + CREATE TABLE IF NOT EXISTS mst2_native_head ( + namespace text PRIMARY KEY CHECK (namespace = '/'), + instance_id text NOT NULL, + sequence bigint NOT NULL CHECK (sequence >= 0), + writer_epoch bigint NOT NULL CHECK (writer_epoch > 0), + root_commit text NOT NULL, + root_tree text NOT NULL, + state text NOT NULL CHECK (state IN ('INITIALIZING', 'READY')), + certificate_receipt_id bigint REFERENCES mst2_native_publication(receipt_id), + CHECK ((state = 'READY') = (certificate_receipt_id IS NOT NULL)) + ); + DO $$ BEGIN + ALTER TABLE mst2_publication ADD CONSTRAINT mst2_native_certificate_version_valid + CHECK (native_certificate_version IS NULL OR native_certificate_version > 0); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + DO $$ BEGIN + ALTER TABLE push_queue ADD CONSTRAINT mst2_expected_native_token_valid CHECK ( + (expected_native_sequence IS NULL AND expected_native_epoch IS NULL + AND expected_native_certificate IS NULL) OR + (expected_native_sequence IS NOT NULL AND expected_native_sequence >= 0 + AND expected_native_epoch IS NOT NULL AND expected_native_epoch > 0) + ); + EXCEPTION WHEN duplicate_object THEN NULL; END $$; + "#, + ) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Historical native certificates and queue fences are never erased. + Ok(()) + } +} diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 822a1f72..3f2c2332 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -139,6 +139,8 @@ mod m20260921_000100_fix_git_tag_unique; mod m20260923_000100_import_repo_cleanups; mod m20260923_000200_canonicalize_import_repo_paths; mod m20260925_000100_media_paging; +mod m20261005_000100_add_mst2_publication_request_digest; +mod m20261005_000200_add_mst2_native_head; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -266,6 +268,8 @@ impl MigratorTrait for Migrator { Box::new(m20260923_000100_import_repo_cleanups::Migration), Box::new(m20260923_000200_canonicalize_import_repo_paths::Migration), Box::new(m20260925_000100_media_paging::Migration), + Box::new(m20261005_000100_add_mst2_publication_request_digest::Migration), + Box::new(m20261005_000200_add_mst2_native_head::Migration), ] } } diff --git a/src/jupiter/service/native_publication_push_tests.rs b/src/jupiter/service/native_publication_push_tests.rs new file mode 100644 index 00000000..9f77b988 --- /dev/null +++ b/src/jupiter/service/native_publication_push_tests.rs @@ -0,0 +1,248 @@ +use super::*; + +use bytes::Bytes; + +use crate::callisto::{ + mst2_native_head, mst2_native_publication, mst2_publication, mst2_publication_outbox, +}; +use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, Statement}; + +const NATIVE_INSTANCE: &str = "6ab219b0-4275-45ba-9d7b-7b0b633018cd"; + +async fn native_fixture() -> (tempfile::TempDir, crate::jupiter::storage::Storage, git_internal::internal::object::commit::Commit, String) { + let temp = tempfile::tempdir().unwrap(); + let mut config = crate::config::testing::isolated_config(temp.path().join("config")); + config.monorepo.push_policy = PushPolicy::Trunk; + config.mst2.enabled = true; + config.mst2.publication_enabled = true; + config.mst2.instance_uuid = Some(NATIVE_INSTANCE.to_owned()); + let storage = crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; + let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; + let name = format!("native-{}", uuid::Uuid::new_v4().simple()); + let (tip, path) = wh03_path_fixture(&storage, &name).await; + // Persist the literal file bytes expected by the tree fixture for actual HTTP projection. + for (oid, raw) in [("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", b"keep".as_slice()), + ("dddddddddddddddddddddddddddddddddddddddd", b"old".as_slice())] { + storage.git_service.save_object_from_raw(Bytes::copy_from_slice(raw)).await.unwrap(); + storage.mono_storage().get_connection().execute_raw(Statement::from_sql_and_values( + storage.mono_storage().get_connection().get_database_backend(), + "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) \ + VALUES ('git',$1,'blob',$2,$3,2,'VERIFIED',now()) ON CONFLICT DO NOTHING", + [oid.into(), Sha256::digest(raw).to_vec().into(), (raw.len() as i64).into()], + )).await.unwrap(); + } + storage.mono_storage().initialize_native_publication(NATIVE_INSTANCE).await.unwrap(); + (temp, storage, tip, path) +} + +async fn save_same_tree_commit(storage: &crate::jupiter::storage::Storage, parent: &git_internal::internal::object::commit::Commit) -> (String, PushPayload) { + let commit = git_internal::internal::object::commit::Commit::from_tree_id(parent.tree_id, vec![parent.id], "same native tree n1"); + storage.mono_storage().save_mega_commits(vec![commit.clone()], None).await.unwrap(); + let oid = commit.id.to_string(); + (oid.clone(), PushPayload { commits: vec![oid], fork_base: Some(parent.id.to_string()), n: 1 }) +} + +#[tokio::test] +async fn real_same_tree_push_advances_one_global_certificate_and_n0_replay_stays_fixed() { + let (_temp, storage, tip, path) = native_fixture().await; + let mono = storage.mono_storage(); + let before = mono.get_main_ref("/").await.unwrap().unwrap(); + let (new, payload) = save_same_tree_commit(&storage, &tip).await; + let id = wh03_enqueue_push(&storage, &path, &tip.id.to_string(), &new, &payload).await; + assert!(matches!(wh03_exec(&storage,id).await, ExecuteOutcome::Done { root_cas_writes:1,.. })); + let after = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!((after.ref_commit_hash,after.ref_tree_hash),(before.ref_commit_hash.clone(),before.ref_tree_hash.clone())); + let head = mono.read_native_publication_head(NATIVE_INSTANCE).await.unwrap(); + assert_eq!(head.token.sequence,1); + let receipt = mst2_publication::Entity::find().filter(mst2_publication::Column::OperationId.eq(format!("mst2:trunk-queue:{id}"))) + .one(mono.get_connection()).await.unwrap().unwrap(); + assert_eq!(receipt.namespace,path); + assert_eq!(receipt.new_oid,new); + assert_eq!(receipt.native_certificate_version,Some(1)); + assert_ne!(receipt.new_oid,head.root.commit); + let noop = PushPayload { commits:vec![],fork_base:None,n:0 }; + let noop_id = wh03_enqueue_push(&storage,&path,&new,&new,&noop).await; + assert!(matches!(wh03_exec(&storage,noop_id).await, ExecuteOutcome::Done { root_cas_writes:1,.. })); + assert_eq!(mono.read_native_publication_head(NATIVE_INSTANCE).await.unwrap().token,head.token); + mono.get_connection().execute_raw(Statement::from_sql_and_values(mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status='Running',landed_commit_id=NULL WHERE id=$1",[id.into()], + )).await.unwrap(); + assert!(matches!(wh03_exec(&storage,id).await, ExecuteOutcome::Done { root_cas_writes:0,.. })); + assert_eq!(mono.read_native_publication_head(NATIVE_INSTANCE).await.unwrap().token,head.token); + assert_eq!(mst2_native_publication::Entity::find().count(mono.get_connection()).await.unwrap(),1); +} + +#[tokio::test] +async fn real_merge_fails_closed_when_native_publication_is_enabled() { + let (_temp, storage, tip, path) = native_fixture().await; + let mono = storage.mono_storage(); + let root_before = mono.get_main_ref("/").await.unwrap().unwrap(); + let head_before = mst2_native_head::Entity::find_by_id("/") + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + + let cl_link = format!("CL-NATIVE-GUARD-{}", uuid::Uuid::new_v4().simple()); + let EnqueueOutcome::Inserted { id } = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Merge, + operation_id: merge_operation_id(&cl_link), + path: path.clone(), + old_id: tip.id.to_string(), + new_id: tip.id.to_string(), + requester: Some("native-guard-test".to_owned()), + payload: serde_json::json!({ + "cl_link": cl_link, + "authz_principal": "native-guard-test", + "execution_actor": "native-guard-test" + }), + ref_name: None, + is_delete: false, + }) + .await + .unwrap() + else { + panic!("merge enqueue did not insert"); + }; + assert_eq!( + storage + .push_queue_service + .storage() + .claim_for_execution(id) + .await + .unwrap(), + ClaimOutcome::Claimed + ); + + let merge_ctx = MergeExecContext { + storage: storage.clone(), + git_object_cache: Arc::new(crate::ceres::api_service::cache::GitObjectCache { + connection: crate::jupiter::tests::test_redis_manager().await, + prefix: String::new(), + }), + abort_before_cl_status: false, + pause_after_apply: Duration::ZERO, + pause_after_apply_barrier: None, + }; + let outcome = storage + .push_queue_service + .execute_b3( + ExecuteRequest { + id, + ..Default::default() + }, + None, + Some(&merge_ctx), + None, + ) + .await + .unwrap(); + assert!(matches!( + outcome, + ExecuteOutcome::Failed { + ref failure, + ref message, + .. + } if failure == "Conflict" + && message == "MST2 native publication does not cover merge writer" + )); + assert_eq!(wh03_row_status(&storage, id).await, PushQueueStatusEnum::Failed); + + let root_after = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!(root_after.ref_commit_hash, root_before.ref_commit_hash); + assert_eq!(root_after.ref_tree_hash, root_before.ref_tree_hash); + let head_after = mst2_native_head::Entity::find_by_id("/") + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(head_after, head_before); + assert_eq!(mst2_publication::Entity::find().count(mono.get_connection()).await.unwrap(), 0); + assert_eq!( + mst2_native_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn real_push_rejects_same_root_changed_publication_token_before_any_ref_write() { + let (_temp, storage, tip, path) = native_fixture().await; + let (new,payload)=save_same_tree_commit(&storage,&tip).await; + let id=wh03_enqueue_push(&storage,&path,&tip.id.to_string(),&new,&payload).await; + let mono=storage.mono_storage(); + // A separate writer transaction changes only the publication generation. + mono.get_connection().execute_unprepared("UPDATE mst2_native_head SET sequence=1 WHERE namespace='/'").await.unwrap(); + let outcome=wh03_exec(&storage,id).await; + assert!(matches!(outcome,ExecuteOutcome::Failed { ref failure,ref message,.. } + if failure=="Conflict" && message.contains("publication token is stale"))); + assert_eq!(mono.get_main_ref(&path).await.unwrap().unwrap().ref_commit_hash,tip.id.to_string()); + assert_eq!(mst2_native_publication::Entity::find().count(mono.get_connection()).await.unwrap(),0); + assert_eq!(mst2_publication::Entity::find().count(mono.get_connection()).await.unwrap(),0); +} + +async fn api_state(storage: crate::jupiter::storage::Storage) -> crate::api::MonoApiServiceState { + crate::api::MonoApiServiceState { + entity_store: storage.entity_store.clone(),storage, + session_store:crate::api::oauth::api_store::BrowserSessionStore::Anonymous, + git_object_cache:Arc::new(crate::ceres::api_service::cache::GitObjectCache { + connection:crate::jupiter::tests::test_redis_manager().await,prefix:String::new(), + }),listen_addr:"127.0.0.1:0".to_owned(), + } +} + +async fn http_resolve(state: crate::api::MonoApiServiceState,scope:&str) -> (u16,serde_json::Value) { + use tower::ServiceExt; + let app=crate::api::router::snapshot_router::routers(state.clone()).with_state(state); + let response=app.oneshot(axum::http::Request::builder().method("POST").uri("/snapshots/resolve") + .header("content-type","application/json").body(axum::body::Body::from(serde_json::json!({"target":{"kind":"latest"},"scope":scope}).to_string())).unwrap()).await.unwrap(); + let status=response.status().as_u16(); + let body=axum::body::to_bytes(response.into_body(),1_048_576).await.unwrap(); + (status,serde_json::from_slice(&body).unwrap()) +} + +#[tokio::test] +async fn http_resolve_captured_before_commit_never_mixes_new_sequence_with_old_descriptor() { + let (_temp,storage,tip,path)=native_fixture().await; + let (first,payload)=save_same_tree_commit(&storage,&tip).await; + let id=wh03_enqueue_push(&storage,&path,&tip.id.to_string(),&first,&payload).await; + assert!(matches!(wh03_exec(&storage,id).await,ExecuteOutcome::Done{..})); + let state=api_state(storage.clone()).await; + let (status,v1)=http_resolve(state.clone(),"/").await; + assert_eq!(status,200); + let (_,subscope)=http_resolve(state.clone(),&path).await; + assert_eq!(v1["publication_sequence"],subscope["publication_sequence"]); + assert_eq!(v1["writer_epoch"],subscope["writer_epoch"]); + let root_v1=storage.mono_storage().get_main_ref("/").await.unwrap().unwrap(); + let _ = root_v1; + let captured=Arc::new(tokio::sync::Barrier::new(2)); + let release=Arc::new(tokio::sync::Barrier::new(2)); + let task_state=state.clone();let task_captured=captured.clone();let task_release=release.clone(); + let mut reader=tokio::spawn(async move { + crate::api::router::snapshot_router::with_native_resolve_barriers(task_captured,task_release,http_resolve(task_state,"/")).await + }); + tokio::time::timeout(Duration::from_secs(5),captured.wait()).await.unwrap(); + let first_commit=storage.mono_storage().get_commit_by_hash(&first).await.unwrap().unwrap(); + let first_commit=git_internal::internal::object::commit::Commit::from_mega_model(first_commit); + let (next,next_payload)=wh03_save_n1_commit(&storage,first_commit.id,"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","next projection").await; + let id=wh03_enqueue_push(&storage,&path,&first,&next,&next_payload).await; + assert!(matches!(wh03_exec(&storage,id).await,ExecuteOutcome::Done{..})); + tokio::time::timeout(Duration::from_secs(5),release.wait()).await.unwrap(); + let (status,delayed)=tokio::time::timeout(Duration::from_secs(10),&mut reader).await.unwrap().unwrap(); + assert_eq!(status,200);assert_eq!(delayed["descriptor"],v1["descriptor"]); + assert_eq!(delayed["publication_sequence"],v1["publication_sequence"]); + let (status,v2)=http_resolve(state,"/").await; + assert_eq!(status,200);assert_eq!(v2["publication_sequence"],"2"); + assert_ne!(v2["descriptor"]["snapshot_id"],v1["descriptor"]["snapshot_id"]); +} diff --git a/src/jupiter/service/push_queue_service.rs b/src/jupiter/service/push_queue_service.rs index 9d563268..be66fcfd 100644 --- a/src/jupiter/service/push_queue_service.rs +++ b/src/jupiter/service/push_queue_service.rs @@ -29,6 +29,11 @@ use crate::{ blob_path_index::BlobPathIndexMode, git_db_storage::GitDbStorage, mono_storage::MonoStorage, + mst2_publication_storage::{ + PreparedPublication, PublicationPreparation, PublicationReceiptError, + PublicationRequest, + }, + native_publication_storage::PreparedNativePublication, push_queue_storage::{ ClaimOutcome, EnqueueOutcome, EnqueueParams, EnqueueRejectReason, PushQueueStorage, }, @@ -416,7 +421,7 @@ pub struct PushExecContext { pub storage: crate::jupiter::storage::Storage, pub git_object_cache: std::sync::Arc, /// Test-only: two-phase sync inside the B3 txn right before - /// `apply_push_in_txn` (after the baseline read): the executor signals + /// `apply_push_in_txn` or the n=0 root CAS (after the baseline read): the executor signals /// "race window open" on the enter barrier, then blocks on the release /// barrier until the test's queue-bypassing writer has committed — a /// deterministic apply-time root CAS miss (WH-03). @@ -425,6 +430,12 @@ pub struct PushExecContext { pub pre_apply_release_barrier: Option>, } +struct PushRefBaseline<'a> { + cur_commit: Option<&'a str>, + cur_tree: Option<&'a str>, + root: Option<&'a crate::callisto::mega_refs::Model>, +} + /// Context required to execute a claimed `kind=merge` round under B3. pub struct MergeExecContext { pub storage: crate::jupiter::storage::Storage, @@ -692,6 +703,11 @@ impl PushQueueService { self } + pub(crate) fn with_native_publication(mut self, enabled: bool) -> Self { + self.push_queue_storage = self.push_queue_storage.with_native_publication(enabled); + self + } + pub fn with_max_push_commits(mut self, max_push_commits: usize) -> Self { self.max_push_commits = max_push_commits; self @@ -1264,8 +1280,56 @@ impl PushQueueService { return Ok(ExecuteOutcome::HardStopped { id: req.id }); } + // Receipt replay precedes baseline/ref writes: the root may already + // have advanced since a committed operation lost its response. + let publication = if let Some(ctx) = push_ctx + && ctx.storage.config().mst2.publication_enabled + && row.kind != PushQueueKindEnum::Attach + && !(row.kind == PushQueueKindEnum::Merge && merge_ctx.is_some()) + { + let request = match PublicationRequest::from_trunk_queue(&row) { + Ok(request) => request, + Err(error) => { + return self + .b3_fail_merge(txn, req.id, "Conflict", error.to_string()) + .await; + } + }; + let preparation = match self + .mono_storage + .begin_publication_in_txn(&txn, request) + .await + { + Ok(preparation) => preparation, + Err(PublicationReceiptError::Database(error)) => return Err(error.into()), + Err(error) => { + return self + .b3_fail_merge(txn, req.id, "Conflict", error.to_string()) + .await; + } + }; + match preparation { + PublicationPreparation::Prepared(prepared) => Some(prepared), + PublicationPreparation::AlreadyCommitted(committed) => { + return self + .b3_replay_committed_operation(txn, req.id, committed.receipt.new_oid) + .await; + } + PublicationPreparation::AlreadyCommittedNoop(receipt) => { + return self + .b3_replay_committed_operation(txn, req.id, receipt.landed_commit_id) + .await; + } + } + } else { + None + }; + // Expected-root baseline compare (NULL-safe). - let root = self.mono_storage.get_main_ref_in_txn("/", &txn).await?; + let root = self + .mono_storage + .get_native_main_ref_in_txn("/", &txn) + .await?; let (cur_commit, cur_tree) = match &root { Some(r) => ( Some(r.ref_commit_hash.as_str()), @@ -1296,6 +1360,54 @@ impl PushQueueService { return Ok(ExecuteOutcome::BypassDetected { id: req.id }); } + // Native publication currently has no merge projection. Refuse the + // real merge writer before it can mutate refs, commits, or CL state; + // keeping this gate ahead of dispatch makes publication fail closed + // while the writer matrix is still incomplete. + if row.kind == PushQueueKindEnum::Merge + && merge_ctx.is_some_and(|ctx| ctx.storage.config().mst2.publication_enabled) + { + return self + .b3_fail_merge( + txn, + req.id, + "Conflict", + "MST2 native publication does not cover merge writer".into(), + ) + .await; + } + + let native_publication = if publication.is_some() && row.kind == PushQueueKindEnum::Push { + let ctx = push_ctx.ok_or_else(|| { + MegaError::Other("native publication requires the real push writer".into()) + })?; + let Some(instance) = ctx.storage.config().mst2.instance_uuid.as_deref() else { + return self + .b3_fail_merge( + txn, + req.id, + "Conflict", + "native publication instance missing".into(), + ) + .await; + }; + match self + .mono_storage + .reserve_native_publication_in_txn(&txn, &row, instance) + .await + { + Ok(prepared) => Some(prepared), + Err(PublicationReceiptError::Database(error)) => return Err(error.into()), + Err(error) => { + return self + .b3_fail_merge(txn, req.id, "Conflict", error.to_string()) + .await; + } + } + } else { + None + }; + if req.force_conflict() { return self .b4_conflict_requeue_holding_lock(txn, req.id, &row) @@ -1346,7 +1458,18 @@ impl PushQueueService { } if let Some(ctx) = push_ctx { return self - .b3_execute_push(txn, &row, ctx, cur_commit, cur_tree, root.as_ref()) + .b3_execute_push( + txn, + &row, + ctx, + PushRefBaseline { + cur_commit, + cur_tree, + root: root.as_ref(), + }, + publication, + native_publication, + ) .await; } tracing::trace!(policy = ?self.push_policy, id = req.id, "B3 push stub"); @@ -1449,26 +1572,16 @@ impl PushQueueService { return Ok(self.outcome_claim_lost(req.id)); } - // T05 (spec 09 §1): record the publication receipt, outbox event and - // per-namespace sequence in the *same* transaction as the root CAS - // above, so the visible change, its sequence and its outbox commit - // atomically. Gated off by default (`mst2.publication_enabled`), so - // an unconfigured deployment keeps the exact prior behavior. - if let Some(ctx) = push_ctx - && ctx.storage.config().mst2.publication_enabled - { - let old_oid = cur_commit.unwrap_or(ZERO_ID); - let namespace = self.mono_storage.normalize_namespace(&row.path); + if let Some(prepared) = publication { self.mono_storage .record_publication_in_txn( &txn, - &row.operation_id, - &namespace, - old_oid, + prepared, + cur_commit.unwrap_or(ZERO_ID), &new_commit, - "trunk_push", ) - .await?; + .await + .map_err(|error| MegaError::Other(error.to_string()))?; } PushQueueStorage::notify_mono_write_queue(&txn).await?; @@ -1481,6 +1594,25 @@ impl PushQueueService { }) } + async fn b3_replay_committed_operation( + &self, + txn: sea_orm::DatabaseTransaction, + id: i64, + landed_commit_id: String, + ) -> Result { + if !PushQueueStorage::mark_done_if_running_in_txn(&txn, id, &landed_commit_id).await? { + txn.rollback().await?; + return Ok(self.outcome_claim_lost(id)); + } + PushQueueStorage::notify_mono_write_queue(&txn).await?; + txn.commit().await?; + Ok(ExecuteOutcome::Done { + id, + landed_commit_id, + root_cas_writes: 0, + }) + } + /// TP-08: attach kind under held `MONO_WRITE_LOCK` — sole root write is /// `attach_to_monorepo_parent_in_txn` (no generic CAS stub, no retry loop). /// @@ -2308,13 +2440,13 @@ impl PushQueueService { txn: sea_orm::DatabaseTransaction, row: &push_queue::Model, ctx: &PushExecContext, - cur_commit: Option<&str>, - cur_tree: Option<&str>, - root: Option<&crate::callisto::mega_refs::Model>, + baseline: PushRefBaseline<'_>, + publication: Option, + native_publication: Option, ) -> Result { let id = row.id; match self - .b3_execute_push_inner(txn, row, ctx, cur_commit, cur_tree, root) + .b3_execute_push_inner(txn, row, ctx, baseline, publication, native_publication) .await { Ok(outcome) => Ok(outcome), @@ -2357,10 +2489,15 @@ impl PushQueueService { txn: sea_orm::DatabaseTransaction, row: &push_queue::Model, ctx: &PushExecContext, - cur_commit: Option<&str>, - cur_tree: Option<&str>, - root: Option<&crate::callisto::mega_refs::Model>, + baseline: PushRefBaseline<'_>, + publication: Option, + native_publication: Option, ) -> Result { + let PushRefBaseline { + cur_commit, + cur_tree, + root, + } = baseline; use std::path::PathBuf; use git_internal::{ @@ -2410,7 +2547,7 @@ impl PushQueueService { let path_row = self .mono_storage - .get_main_ref_in_txn(&row.path, &txn) + .get_native_main_ref_in_txn(&row.path, &txn) .await?; if path_row.is_none() { @@ -2481,6 +2618,12 @@ impl PushQueueService { ) .await; } + if let Some(barrier) = &ctx.pre_apply_enter_barrier { + barrier.wait().await; + } + if let Some(barrier) = &ctx.pre_apply_release_barrier { + barrier.wait().await; + } PushQueueStorage::savepoint(&txn, "b3_kind").await?; let cas_ok = self .mono_storage @@ -2517,6 +2660,24 @@ impl PushQueueService { txn.rollback().await?; return Ok(self.outcome_claim_lost(row.id)); } + if let Some(prepared) = native_publication { + self.mono_storage + .finish_native_noop_in_txn(&txn, prepared) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } + if let Some(prepared) = publication { + self.mono_storage + .record_noop_operation_in_txn( + &txn, + prepared, + &root.ref_commit_hash, + &root.ref_tree_hash, + &pref.ref_commit_hash, + ) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } PushQueueStorage::notify_mono_write_queue(&txn).await?; txn.commit().await?; self.run_c_segment_index(row.id, &row.path).await; @@ -2786,26 +2947,23 @@ impl PushQueueService { return Ok(self.outcome_claim_lost(row.id)); } - // T05 (spec 09 §1/§7): record the publication receipt, outbox event - // and per-namespace sequence in the *same* transaction as the ref CAS - // above, so the visible change, its sequence and its outbox commit - // atomically (trunk push = the primary default-namespace writer of - // spec 09 §5). Gated off by default (`mst2.publication_enabled`): - // a deployment that has not passed the §9 shadow comparison keeps - // the exact prior behavior. Replaying the same operation id lands - // the receipt once (PUB-11). - if ctx.storage.config().mst2.publication_enabled { - let namespace = self.mono_storage.normalize_namespace(&normalized); - self.mono_storage + if let Some(prepared) = publication { + let committed = self + .mono_storage .record_publication_in_txn( &txn, - &row.operation_id, - &namespace, + prepared, cur_commit.unwrap_or(ZERO_ID), &landed_commit_id, - "trunk_push", ) - .await?; + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + let native = native_publication + .ok_or_else(|| MegaError::Other("native push reservation missing".into()))?; + self.mono_storage + .record_native_publication_in_txn(&txn, native, &committed) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; } PushQueueStorage::notify_mono_write_queue(&txn).await?; @@ -3388,6 +3546,7 @@ mod tests { use serde_json::json; use super::*; + include!("native_publication_push_tests.rs"); use crate::{ callisto::{mega_refs, sea_orm_active_enums::PushQueueFailureEnum}, jupiter::{ @@ -4804,6 +4963,503 @@ mod tests { .unwrap() } + #[tokio::test] + async fn mst2_receipt_real_push_replays_before_root_baseline_and_rejects_changed_request() { + use sea_orm::{ + ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, Statement, + }; + + use crate::callisto::{mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt}; + + let temp = tempfile::tempdir().unwrap(); + let mut config = crate::config::testing::isolated_config(temp.path().join("config")); + config.monorepo.push_policy = PushPolicy::Trunk; + config.mst2.publication_enabled = true; + config.mst2.instance_uuid = Some("6ab219b0-4275-45ba-9d7b-7b0b633018cd".to_owned()); + let storage = crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; + let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; + let (old_commit, path) = wh03_path_fixture(&storage, "receipt").await; + storage + .mono_storage() + .initialize_native_publication("6ab219b0-4275-45ba-9d7b-7b0b633018cd") + .await + .unwrap(); + let old_id = old_commit.id.to_string(); + let (new_id, payload) = wh03_save_n1_commit( + &storage, + old_commit.id, + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "receipt new", + ) + .await; + let id = wh03_enqueue_push(&storage, &path, &old_id, &new_id, &payload).await; + assert!(matches!( + wh03_exec(&storage, id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let mono = storage.mono_storage(); + let root_after = mono.get_main_ref("/").await.unwrap().unwrap(); + let operation_id = format!("mst2:trunk-queue:{id}"); + let receipt = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(&operation_id)) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.request_digest_version, Some(1)); + assert!(receipt.request_digest.is_some()); + assert_eq!(receipt.new_oid, new_id); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + let retry = |requester: Option, payload: JsonValue| EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: push_operation_id(&old_id, &new_id), + path: path.clone(), + old_id: old_id.clone(), + new_id: new_id.clone(), + requester, + payload, + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }; + assert!( + matches!(storage.push_queue_service.enqueue(retry(None, payload.to_json())).await.unwrap(), + EnqueueOutcome::Replay { id: replay_id, .. } if replay_id == id) + ); + let error = storage + .push_queue_service + .enqueue(retry(Some("different-actor".to_owned()), payload.to_json())) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + let mut different = payload.to_json(); + different["fork_base"] = serde_json::json!("different-base"); + let error = storage + .push_queue_service + .enqueue(retry(None, different)) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + + // Model recovery of a Running queue row after COMMIT lost its response. + // Original claim baselines remain old, so a late replay would fail them. + mono.get_connection() + .execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', landed_commit_id = NULL WHERE id = $1", + [id.into()], + )) + .await + .unwrap(); + assert_eq!( + wh03_exec(&storage, id).await, + ExecuteOutcome::Done { + id, + landed_commit_id: new_id.clone(), + root_cas_writes: 0, + } + ); + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!(root.ref_commit_hash, root_after.ref_commit_hash); + assert_eq!(root.ref_tree_hash, root_after.ref_tree_hash); + let check_txn = mono.get_connection().begin().await.unwrap(); + assert!( + !PushQueueStorage::is_hard_stopped_in_txn(&check_txn) + .await + .unwrap() + ); + check_txn.rollback().await.unwrap(); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', requester = 'changed-actor' WHERE id = $1", + [id.into()], + )).await.unwrap(); + assert!( + matches!(wh03_exec(&storage, id).await, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("MST2_PUBLICATION_CONFLICT")) + ); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + root_after.ref_commit_hash + ); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + + // Distinct internal no-op operations do not create publications. + let n0_payload = PushPayload { + commits: Vec::new(), + fork_base: None, + n: 0, + }; + let mut noop_ids = Vec::new(); + for round in 0..3 { + let operation_id = format!("internal-noop-{round}"); + let outcome = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: operation_id.clone(), + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: None, + payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap(); + let EnqueueOutcome::Inserted { id: n0_id } = outcome else { + panic!("{outcome:?}"); + }; + assert_eq!( + storage + .push_queue_service + .storage() + .claim_for_execution(n0_id) + .await + .unwrap(), + ClaimOutcome::Claimed + ); + assert!(matches!( + wh03_exec(&storage, n0_id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let noop = mst2_queue_noop_receipt::Entity::find() + .filter( + mst2_queue_noop_receipt::Column::OperationId + .eq(format!("mst2:trunk-queue:{n0_id}")), + ) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(noop.observed_sequence, 1); + assert_eq!(noop.observed_root_commit, root_after.ref_commit_hash); + assert_eq!(noop.observed_root_tree, root_after.ref_tree_hash); + assert_eq!(noop.landed_commit_id, new_id); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 1); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + assert!(matches!(storage.push_queue_service.enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, operation_id: operation_id.clone(), + path: path.clone(), old_id: new_id.clone(), new_id: new_id.clone(), + requester: None, payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), is_delete: false, + }).await.unwrap(), EnqueueOutcome::Replay { id: replay_id, .. } if replay_id == n0_id)); + let error = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id: operation_id.clone(), + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: Some("changed-actor".to_owned()), + payload: n0_payload.to_json(), + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + let mut changed_payload = n0_payload.to_json(); + changed_payload["fork_base"] = serde_json::json!("changed-intent"); + let error = storage + .push_queue_service + .enqueue(EnqueueRequest { + kind: PushQueueKindEnum::Push, + operation_id, + path: path.clone(), + old_id: new_id.clone(), + new_id: new_id.clone(), + requester: None, + payload: changed_payload, + ref_name: Some(MEGA_BRANCH_NAME.to_owned()), + is_delete: false, + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("MST2_PUBLICATION_CONFLICT")); + noop_ids.push(n0_id); + } + let (later_id, later_payload) = wh03_save_n1_commit( + &storage, + new_id.parse().unwrap(), + "ffffffffffffffffffffffffffffffffffffffff", + "after no-op", + ) + .await; + let later_queue_id = + wh03_enqueue_push(&storage, &path, &new_id, &later_id, &later_payload).await; + assert!(matches!( + wh03_exec(&storage, later_queue_id).await, + ExecuteOutcome::Done { + root_cas_writes: 1, + .. + } + )); + let later_root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_ne!(later_root.ref_commit_hash, root_after.ref_commit_hash); + for &n0_id in &noop_ids { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', landed_commit_id = NULL WHERE id = $1", + [n0_id.into()], + )).await.unwrap(); + assert_eq!( + wh03_exec(&storage, n0_id).await, + ExecuteOutcome::Done { + id: n0_id, + landed_commit_id: new_id.clone(), + root_cas_writes: 0, + } + ); + } + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!(root.ref_commit_hash, later_root.ref_commit_hash); + assert_eq!(root.ref_tree_hash, later_root.ref_tree_hash); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 2); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 3 + ); + let n0_id = noop_ids[0]; + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET status = 'Running', requester = 'changed-actor' WHERE id = $1", + [n0_id.into()], + )).await.unwrap(); + assert!( + matches!(wh03_exec(&storage, n0_id).await, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("MST2_PUBLICATION_CONFLICT")) + ); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + later_root.ref_commit_hash + ); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 2); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 3 + ); + } + + #[tokio::test] + async fn mst2_noop_real_push_checks_epoch_baseline_and_net_zero_cas_fences() { + use sea_orm::{ConnectionTrait, EntityTrait, PaginatorTrait, Statement}; + + use crate::callisto::{mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt}; + + for fence in ["epoch", "baseline", "cas"] { + let temp = tempfile::tempdir().unwrap(); + let mut config = crate::config::testing::isolated_config(temp.path().join("config")); + config.monorepo.push_policy = PushPolicy::Trunk; + config.mst2.publication_enabled = true; + config.mst2.instance_uuid = Some("6ab219b0-4275-45ba-9d7b-7b0b633018cd".to_owned()); + let storage = + crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; + let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; + let (tip, path) = wh03_path_fixture(&storage, &format!("noop-{fence}")).await; + storage + .mono_storage() + .initialize_native_publication("6ab219b0-4275-45ba-9d7b-7b0b633018cd") + .await + .unwrap(); + let tip_id = tip.id.to_string(); + let payload = PushPayload { + commits: Vec::new(), + fork_base: None, + n: 0, + }; + let id = wh03_enqueue_push(&storage, &path, &tip_id, &tip_id, &payload).await; + let mono = storage.mono_storage(); + let root_before = mono.get_main_ref("/").await.unwrap().unwrap(); + let outcome = if fence == "epoch" { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ($1, 0, 2)", + [path.clone().into()], + )).await.unwrap(); + wh03_exec(&storage, id).await + } else if fence == "baseline" { + mono.get_connection() + .execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE push_queue SET expected_commit_hash = $1 WHERE id = $2", + ["f".repeat(40).into(), id.into()], + )) + .await + .unwrap(); + wh03_exec(&storage, id).await + } else { + let enter = Arc::new(tokio::sync::Barrier::new(2)); + let release = Arc::new(tokio::sync::Barrier::new(2)); + let ctx = PushExecContext { + storage: storage.clone(), + git_object_cache: Arc::new(crate::ceres::api_service::cache::GitObjectCache { + connection: crate::jupiter::tests::test_redis_manager().await, + prefix: String::new(), + }), + pre_apply_enter_barrier: Some(Arc::clone(&enter)), + pre_apply_release_barrier: Some(Arc::clone(&release)), + }; + let exec_storage = storage.clone(); + let mut worker = tokio::spawn(async move { + exec_storage + .push_queue_service + .execute_b3( + ExecuteRequest { + id, + ..Default::default() + }, + None, + None, + Some(&ctx), + ) + .await + }); + let handshake = async { + tokio::time::timeout(Duration::from_secs(5), enter.wait()).await.map_err(|_| "no-op did not reach CAS".to_owned())?; + tokio::time::timeout(Duration::from_secs(5), mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE mega_refs SET ref_commit_hash = $1 WHERE path = '/' AND ref_name = $2", + ["f".repeat(40).into(), MEGA_BRANCH_NAME.into()], + ))).await.map_err(|_| "bypass writer timed out".to_owned())?.map_err(|error| error.to_string())?; + tokio::time::timeout(Duration::from_secs(5), release.wait()).await.map_err(|_| "no-op CAS release timed out".to_owned()) + }.await; + if let Err(error) = handshake { + worker.abort(); + let _ = tokio::time::timeout(Duration::from_secs(5), &mut worker).await; + panic!("{error}"); + } + match tokio::time::timeout(Duration::from_secs(10), &mut worker).await { + Ok(result) => result.unwrap().unwrap(), + Err(_) => { + worker.abort(); + let _ = tokio::time::timeout(Duration::from_secs(5), &mut worker).await; + panic!("no-op CAS worker did not finish"); + } + } + }; + if fence == "epoch" { + assert!( + matches!(outcome, ExecuteOutcome::Failed { ref failure, ref message, .. } + if failure == "Conflict" && message.contains("writer epoch is fenced")) + ); + } else { + assert_eq!(outcome, ExecuteOutcome::BypassDetected { id }); + } + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!( + root.ref_commit_hash, + if fence == "cas" { + "f".repeat(40) + } else { + root_before.ref_commit_hash + } + ); + assert_eq!(root.ref_tree_hash, root_before.ref_tree_hash); + assert_eq!(mono.publication_sequence(&path).await.unwrap(), 0); + assert_eq!( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + let txn = mono.get_connection().begin().await.unwrap(); + assert_eq!( + PushQueueStorage::is_hard_stopped_in_txn(&txn) + .await + .unwrap(), + fence != "epoch" + ); + txn.rollback().await.unwrap(); + } + } + /// Same as `wh03_exec`, with an explicit request override (force flags). async fn wh03_exec_with( storage: &crate::jupiter::storage::Storage, diff --git a/src/jupiter/storage/mod.rs b/src/jupiter/storage/mod.rs index 17121c7d..4d060c7c 100644 --- a/src/jupiter/storage/mod.rs +++ b/src/jupiter/storage/mod.rs @@ -16,6 +16,8 @@ pub mod issue_storage; pub mod lfs_db_storage; pub mod media_paging_storage; pub mod mono_storage; +pub(crate) mod mst2_publication_storage; +pub(crate) mod native_publication_storage; pub mod notification_storage; pub mod object_storage; pub mod oci_db_storage; @@ -195,7 +197,8 @@ impl Storage { }; let commit_binding_storage = CommitBindingStorage { base: base.clone() }; - let push_queue_storage = PushQueueStorage::new(base.clone()); + let push_queue_storage = PushQueueStorage::new(base.clone()) + .with_native_publication(config.mst2.publication_enabled); let buck_storage = BuckStorage { base: base.clone() }; let bots_storage = BotsStorage { base: base.clone() }; @@ -260,7 +263,8 @@ impl Storage { }; let push_queue_service = PushQueueService::new(base.clone(), config.monorepo.push_policy.clone()) - .with_max_push_commits(config.monorepo.max_push_commits); + .with_max_push_commits(config.monorepo.max_push_commits) + .with_native_publication(config.mst2.publication_enabled); let artifact_service = ArtifactService::new(base.clone(), object_store.clone()); let buck_service = BuckService::new( base.clone(), diff --git a/src/jupiter/storage/mono_storage.rs b/src/jupiter/storage/mono_storage.rs index dc140b2e..6d7c964f 100644 --- a/src/jupiter/storage/mono_storage.rs +++ b/src/jupiter/storage/mono_storage.rs @@ -26,7 +26,7 @@ use sea_orm::{ use crate::{ callisto::{ mega_blob, mega_cl, mega_commit, mega_ref_tombstones, mega_refs, mega_tag, mega_tree, - mst2_publication, mst2_publication_outbox, mst2_verified_object, + mst2_verified_object, }, common::{ errors::MegaError, @@ -274,6 +274,24 @@ impl MonoStorage { Ok(result) } + pub(crate) async fn get_native_main_ref_in_txn( + &self, + path: &str, + txn: &DatabaseTransaction, + ) -> Result, MegaError> { + let rows = mega_refs::Entity::find() + .filter(mega_refs::Column::Path.eq(path)) + .filter(mega_refs::Column::RefName.eq(MEGA_BRANCH_NAME)) + .filter(mega_refs::Column::IsCl.eq(false)) + .all(txn) + .await?; + match rows.len() { + 0 => Ok(None), + 1 => Ok(rows.into_iter().next()), + _ => Err(MegaError::Other("selected native ref is ambiguous".into())), + } + } + pub async fn get_main_ref_in_txn( &self, path: &str, @@ -1362,6 +1380,7 @@ impl MonoStorage { updated_at = $3 WHERE path = '/' AND ref_name = $4 + AND is_cl = false AND ref_commit_hash IS NOT DISTINCT FROM $5 AND ref_tree_hash IS NOT DISTINCT FROM $6 "#, @@ -1830,112 +1849,6 @@ impl MonoStorage { } } - /// Record one namespace publication inside the caller's transaction - /// (spec 09 §1/§7): bump the per-namespace sequence, insert the - /// receipt and append the outbox event — atomically with the ref CAS - /// the same transaction performs. - /// - /// Idempotent per (namespace, operation_id): a retried writer gets - /// back the original sequence and neither the counter nor the outbox - /// advances twice (PUB-11). The uniqueness is scoped to the namespace - /// because push operation ids (`old→new`) are only unique within one - /// repo: two namespaces landing the same (old,new) pair are distinct - /// publications, not replays. - pub async fn record_publication_in_txn( - &self, - txn: &DatabaseTransaction, - operation_id: &str, - namespace: &str, - old_oid: &str, - new_oid: &str, - writer_kind: &str, - ) -> Result { - // One SELECT by operation id answers both questions: the same id - // under this namespace is a replay (PUB-11) and returns the original - // sequence; the same id under a different namespace means the id is - // not actually unique to this publication — refuse rather than - // silently double-bind it. - if let Some(existing) = mst2_publication::Entity::find() - .filter(mst2_publication::Column::OperationId.eq(operation_id)) - .one(txn) - .await? - { - if existing.namespace == namespace { - return Ok(existing.sequence); - } - return Err(MegaError::Other(format!( - "operation id {operation_id} already published under a different namespace" - ))); - } - // Upsert the sequence row and bump it atomically; the unique - // namespace key makes this a single-winner counter. - let backend = txn.get_database_backend(); - let bump = sea_orm::Statement::from_sql_and_values( - backend, - r#"INSERT INTO mst2_namespace_seq ("namespace", "sequence", "epoch") - VALUES ($1, 1, 1) - ON CONFLICT ("namespace") DO UPDATE SET "sequence" = "mst2_namespace_seq"."sequence" + 1 - RETURNING "sequence""#, - [namespace.into()], - ); - let row = txn - .query_one_raw(bump) - .await? - .ok_or_else(|| MegaError::Other("mst2_namespace_seq upsert returned no row".into()))?; - let sequence: i64 = row.try_get_by_index(0)?; - let now = chrono::Utc::now().fixed_offset(); - - // Concurrency-safe insert: the receipt is unique per - // (namespace, operation_id). The replay check above covers a - // committed prior publication; a truly concurrent same-operation - // writer loses at the outbox insert's unique operation_id below, - // rolling this transaction back rather than double-publishing. - let insert = mst2_publication::ActiveModel { - id: sea_orm::ActiveValue::NotSet, - operation_id: Set(operation_id.to_string()), - namespace: Set(namespace.to_string()), - sequence: Set(sequence), - old_oid: Set(old_oid.to_string()), - new_oid: Set(new_oid.to_string()), - writer_epoch: Set(1), - writer_kind: Set(writer_kind.to_string()), - created_at: Set(now), - }; - match mst2_publication::Entity::insert(insert) - .on_conflict( - OnConflict::columns([ - mst2_publication::Column::Namespace, - mst2_publication::Column::OperationId, - ]) - .do_nothing() - .to_owned(), - ) - .exec(txn) - .await - { - // RecordNotInserted means a receipt with this identity already - // exists; see the outbox insert below for how a racing writer - // of the same operation is resolved. - Ok(_) | Err(DbErr::RecordNotInserted) => {} - Err(e) => return Err(e.into()), - } - - // The replay check above already returned for a pre-existing receipt, - // so reaching here means this call owns `sequence`; the outbox row - // inserts normally. - mst2_publication_outbox::ActiveModel { - id: sea_orm::ActiveValue::NotSet, - operation_id: Set(operation_id.to_string()), - namespace: Set(namespace.to_string()), - sequence: Set(sequence), - state: Set("PENDING".to_string()), - created_at: Set(now), - } - .insert(txn) - .await?; - Ok(sequence) - } - /// Canonical publication namespace for a repository path: a leading /// slash, no trailing slash, root is `/`. Publication identities are /// keyed on this so `/p/` and `/p` never count as two namespaces. @@ -3261,7 +3174,7 @@ mod tests { // Rollback ⇒ nothing visible: the receipt commits only with the // ref CAS in the same transaction (spec 09 §1). let txn = mono.get_connection().begin().await.unwrap(); - mono.record_publication_in_txn(&txn, "op-roll", "/", "", "aaa", "trunk_push") + mono.record_test_publication_in_txn(&txn, "op-roll", "/", "", "aaa", "trunk_push") .await .unwrap(); txn.rollback().await.unwrap(); @@ -3281,7 +3194,7 @@ mod tests { for (op, old, new) in [("op-1", "", "a1"), ("op-2", "a1", "a2")] { let txn = mono.get_connection().begin().await.unwrap(); let seq = mono - .record_publication_in_txn(&txn, op, "/", old, new, "trunk_push") + .record_test_publication_in_txn(&txn, op, "/", old, new, "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3300,7 +3213,7 @@ mod tests { // adds nothing (PUB-11) — the push retry path depends on this. let txn = mono.get_connection().begin().await.unwrap(); let again = mono - .record_publication_in_txn(&txn, "op-1", "/", "", "a1", "trunk_push") + .record_test_publication_in_txn(&txn, "op-1", "/", "", "a1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3317,7 +3230,7 @@ mod tests { // Distinct namespaces keep independent sequences. let txn = mono.get_connection().begin().await.unwrap(); let other = mono - .record_publication_in_txn(&txn, "op-child", "/child", "", "c1", "trunk_push") + .record_test_publication_in_txn(&txn, "op-child", "/child", "", "c1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); @@ -3334,13 +3247,13 @@ mod tests { // replay: refuse rather than silently double-bind the id (the // reviewer's P3.1 — a silent no-op left /b's counter at 0 forever). let txn = mono.get_connection().begin().await.unwrap(); - mono.record_publication_in_txn(&txn, "shared-op", "/", "", "a1", "trunk_push") + mono.record_test_publication_in_txn(&txn, "shared-op", "/", "", "a1", "trunk_push") .await .unwrap(); txn.commit().await.unwrap(); let txn = mono.get_connection().begin().await.unwrap(); let err = mono - .record_publication_in_txn(&txn, "shared-op", "/b", "", "a1", "trunk_push") + .record_test_publication_in_txn(&txn, "shared-op", "/b", "", "a1", "trunk_push") .await; assert!(err.is_err(), "cross-namespace reuse must be refused"); drop(txn); // the failed transaction rolls back on drop @@ -3363,7 +3276,7 @@ mod tests { let m = base.mono_storage(); let txn = m.get_connection().begin().await.unwrap(); let seq = m - .record_publication_in_txn( + .record_test_publication_in_txn( &txn, &format!("op-c{i}"), "/", diff --git a/src/jupiter/storage/mst2_publication_storage.rs b/src/jupiter/storage/mst2_publication_storage.rs new file mode 100644 index 00000000..a7111c44 --- /dev/null +++ b/src/jupiter/storage/mst2_publication_storage.rs @@ -0,0 +1,514 @@ +//! SQL receipt identity for the existing trunk queue adapter. +//! +//! This is not a complete namespace/projection publication implementation. + +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, ConnectionTrait, DatabaseTransaction, DbErr, + EntityTrait, QueryFilter, Statement, +}; +use sha2::{Digest, Sha256}; + +use crate::{ + callisto::{ + mst2_publication, mst2_publication_outbox, mst2_queue_noop_receipt, push_queue, + sea_orm_active_enums::PushQueueKindEnum, + }, + common::utils::MEGA_BRANCH_NAME, + jupiter::storage::mono_storage::MonoStorage, +}; + +const REQUEST_DIGEST_VERSION: i32 = 1; +const WRITER_EPOCH: i64 = 1; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PublicationReceiptError { + #[error("MST2_PUBLICATION_CONFLICT: {0}")] + Conflict(String), + #[error("MST2_PUBLICATION_LEGACY_RECEIPT: operation {0} has no supported request digest")] + LegacyReceipt(String), + #[error("MST2_PUBLICATION_INTEGRITY_ERROR: {0}")] + Integrity(String), + #[error(transparent)] + Database(#[from] DbErr), +} + +/// Constructed from a server-persisted queue row, never a client digest. +#[derive(Debug, Clone)] +pub(crate) struct PublicationRequest { + operation_id: String, + namespace: String, + writer_kind: String, + request_digest: String, + legacy_operation_id: Option, + is_noop: bool, +} + +impl PublicationRequest { + pub(crate) fn from_trunk_queue( + row: &push_queue::Model, + ) -> Result { + if row.id <= 0 || row.operation_id.is_empty() { + return Err(PublicationReceiptError::Integrity( + "invalid queue operation identity".into(), + )); + } + mst2_codec::descriptor::validate_scope(&row.path) + .map_err(|error| PublicationReceiptError::Integrity(error.to_string()))?; + let writer_kind = match &row.kind { + PushQueueKindEnum::Push => "trunk_push", + PushQueueKindEnum::Merge => "trunk_merge_stub", + PushQueueKindEnum::Attach => { + return Err(PublicationReceiptError::Integrity( + "attach is not covered by this adapter".into(), + )); + } + }; + let operation_id = format!("mst2:trunk-queue:{}", row.id); + let mut hash = request_hasher(&operation_id, &row.path, writer_kind); + hash_field(&mut hash, row.operation_id.as_bytes()); + hash_field(&mut hash, MEGA_BRANCH_NAME.as_bytes()); + hash_field(&mut hash, row.old_id.as_bytes()); + hash_field(&mut hash, row.new_id.as_bytes()); + hash.update([u8::from(row.requester.is_some())]); + if let Some(actor) = &row.requester { + hash_field(&mut hash, actor.as_bytes()); + } + hash_json(&mut hash, &row.payload); + Ok(Self { + operation_id, + namespace: row.path.clone(), + writer_kind: writer_kind.to_string(), + request_digest: format!("sha256:{}", hex::encode(hash.finalize())), + legacy_operation_id: Some(row.operation_id.clone()), + is_noop: row.kind == PushQueueKindEnum::Push + && row.old_id == row.new_id + && row.payload.get("n").and_then(serde_json::Value::as_u64) == Some(0), + }) + } + + #[cfg(test)] + fn for_test( + operation_id: &str, + namespace: &str, + old: &str, + new: &str, + writer_kind: &str, + ) -> Self { + let mut hash = request_hasher(operation_id, namespace, writer_kind); + hash_field(&mut hash, old.as_bytes()); + hash_field(&mut hash, new.as_bytes()); + Self { + operation_id: operation_id.to_owned(), + namespace: namespace.to_owned(), + writer_kind: writer_kind.to_owned(), + request_digest: format!("sha256:{}", hex::encode(hash.finalize())), + legacy_operation_id: None, + is_noop: false, + } + } +} + +fn request_hasher(operation_id: &str, namespace: &str, writer_kind: &str) -> Sha256 { + let mut hash = Sha256::new(); + hash.update(b"mega.mst2.trunk-queue-request\0"); + hash.update(REQUEST_DIGEST_VERSION.to_le_bytes()); + hash.update(WRITER_EPOCH.to_le_bytes()); + hash_field(&mut hash, operation_id.as_bytes()); + hash_field(&mut hash, namespace.as_bytes()); + hash_field(&mut hash, writer_kind.as_bytes()); + hash +} + +fn hash_field(hash: &mut Sha256, bytes: &[u8]) { + hash.update((bytes.len() as u64).to_le_bytes()); + hash.update(bytes); +} + +fn hash_json(hash: &mut Sha256, value: &serde_json::Value) { + use serde_json::Value; + match value { + Value::Null => hash.update([0]), + Value::Bool(value) => hash.update([1, u8::from(*value)]), + Value::Number(value) => { + hash.update([2]); + hash_field(hash, value.to_string().as_bytes()); + } + Value::String(value) => { + hash.update([3]); + hash_field(hash, value.as_bytes()); + } + Value::Array(values) => { + hash.update([4]); + hash.update((values.len() as u64).to_le_bytes()); + for value in values { + hash_json(hash, value); + } + } + Value::Object(values) => { + hash.update([5]); + hash.update((values.len() as u64).to_le_bytes()); + let mut keys: Vec<_> = values.keys().collect(); + keys.sort_unstable(); + for key in keys { + hash_field(hash, key.as_bytes()); + hash_json(hash, &values[key]); + } + } + } +} + +#[derive(Debug)] +pub(crate) struct CommittedPublication { + pub(crate) receipt: mst2_publication::Model, + pub(crate) outbox: mst2_publication_outbox::Model, +} + +#[derive(Debug)] +pub(crate) enum PublicationPreparation { + Prepared(PreparedPublication), + AlreadyCommitted(CommittedPublication), + AlreadyCommittedNoop(mst2_queue_noop_receipt::Model), +} + +/// The SQL transaction id prevents finalizing a reservation in another txn. +#[derive(Debug)] +pub(crate) struct PreparedPublication { + request: PublicationRequest, + sequence: i64, + transaction_id: i64, +} + +async fn validate_committed_publication( + txn: &DatabaseTransaction, + request: &PublicationRequest, + receipt: mst2_publication::Model, +) -> Result { + if receipt.namespace != request.namespace { + return Err(PublicationReceiptError::Conflict( + "operation belongs to another namespace".into(), + )); + } + if receipt.request_digest_version != Some(REQUEST_DIGEST_VERSION) + || receipt.request_digest.is_none() + { + return Err(PublicationReceiptError::LegacyReceipt( + request.operation_id.clone(), + )); + } + if receipt.writer_kind != request.writer_kind + || receipt.writer_epoch != WRITER_EPOCH + || receipt.request_digest.as_deref() != Some(request.request_digest.as_str()) + { + return Err(PublicationReceiptError::Conflict( + "operation request changed".into(), + )); + } + let outbox = mst2_publication_outbox::Entity::find() + .filter(mst2_publication_outbox::Column::OperationId.eq(&request.operation_id)) + .one(txn) + .await? + .ok_or_else(|| PublicationReceiptError::Integrity("receipt outbox missing".into()))?; + if outbox.namespace != receipt.namespace || outbox.sequence != receipt.sequence { + return Err(PublicationReceiptError::Integrity( + "receipt outbox identity mismatch".into(), + )); + } + Ok(CommittedPublication { receipt, outbox }) +} + +async fn reject_legacy_queue_receipt( + txn: &DatabaseTransaction, + legacy_operation_id: Option<&str>, + namespace: &str, +) -> Result<(), PublicationReceiptError> { + if let Some(legacy_id) = legacy_operation_id + && let Some(legacy) = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(legacy_id)) + .filter(mst2_publication::Column::Namespace.eq(namespace)) + .one(txn) + .await? + && (legacy.request_digest_version != Some(REQUEST_DIGEST_VERSION) + || legacy.request_digest.is_none()) + { + return Err(PublicationReceiptError::LegacyReceipt(legacy_id.to_owned())); + } + Ok(()) +} + +enum RecordedOperation { + Publication(mst2_publication::Model), + Noop(mst2_queue_noop_receipt::Model), +} + +async fn find_committed_operation( + txn: &DatabaseTransaction, + operation_id: &str, +) -> Result, PublicationReceiptError> { + let publication = mst2_publication::Entity::find() + .filter(mst2_publication::Column::OperationId.eq(operation_id)) + .one(txn) + .await?; + let noop = mst2_queue_noop_receipt::Entity::find() + .filter(mst2_queue_noop_receipt::Column::OperationId.eq(operation_id)) + .one(txn) + .await?; + match (publication, noop) { + (Some(_), Some(_)) => Err(PublicationReceiptError::Integrity( + "operation has both publication and no-op receipts".into(), + )), + (Some(receipt), None) => Ok(Some(RecordedOperation::Publication(receipt))), + (None, Some(receipt)) => Ok(Some(RecordedOperation::Noop(receipt))), + (None, None) => Ok(None), + } +} + +async fn validate_committed_operation( + txn: &DatabaseTransaction, + request: &PublicationRequest, + existing: RecordedOperation, +) -> Result { + match existing { + RecordedOperation::Publication(receipt) => Ok(PublicationPreparation::AlreadyCommitted( + validate_committed_publication(txn, request, receipt).await?, + )), + RecordedOperation::Noop(receipt) => { + if receipt.namespace != request.namespace + || receipt.writer_kind != request.writer_kind + || receipt.writer_epoch != WRITER_EPOCH + || !request.is_noop + || receipt.request_digest != request.request_digest + { + return Err(PublicationReceiptError::Conflict( + "operation request changed".into(), + )); + } + if receipt.request_digest_version != REQUEST_DIGEST_VERSION { + return Err(PublicationReceiptError::LegacyReceipt( + request.operation_id.clone(), + )); + } + if mst2_publication_outbox::Entity::find() + .filter(mst2_publication_outbox::Column::OperationId.eq(&request.operation_id)) + .one(txn) + .await? + .is_some() + { + return Err(PublicationReceiptError::Integrity( + "no-op operation has a publication outbox".into(), + )); + } + Ok(PublicationPreparation::AlreadyCommittedNoop(receipt)) + } + } +} + +impl MonoStorage { + /// Queue admission can replay Done without executing B3; audit that path too. + pub(crate) async fn validate_queue_publication_replay_in_txn( + &self, + txn: &DatabaseTransaction, + candidate: &push_queue::Model, + ) -> Result<(), PublicationReceiptError> { + let operation_id = format!("mst2:trunk-queue:{}", candidate.id); + let Some(existing) = find_committed_operation(txn, &operation_id).await? else { + reject_legacy_queue_receipt(txn, Some(&candidate.operation_id), &candidate.path) + .await?; + return Ok(()); + }; + let request = PublicationRequest::from_trunk_queue(candidate)?; + if let RecordedOperation::Publication(receipt) = &existing { + self.validate_native_historical_receipt_in_txn(txn, receipt) + .await?; + } + validate_committed_operation(txn, &request, existing).await?; + Ok(()) + } + + /// Lock before any business ref write, then recheck immutable receipt identity. + pub(crate) async fn begin_publication_in_txn( + &self, + txn: &DatabaseTransaction, + request: PublicationRequest, + ) -> Result { + txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ($1, 0, $2) \ + ON CONFLICT (namespace) DO NOTHING", + [request.namespace.clone().into(), WRITER_EPOCH.into()], + )) + .await?; + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT sequence, epoch, txid_current() AS transaction_id \ + FROM mst2_namespace_seq WHERE namespace = $1 FOR UPDATE", + [request.namespace.clone().into()], + )) + .await? + .ok_or_else(|| PublicationReceiptError::Integrity("namespace row missing".into()))?; + let sequence: i64 = row.try_get("", "sequence")?; + let epoch: i64 = row.try_get("", "epoch")?; + if let Some(existing) = find_committed_operation(txn, &request.operation_id).await? { + if let RecordedOperation::Publication(receipt) = &existing { + self.validate_native_historical_receipt_in_txn(txn, receipt) + .await?; + } + return validate_committed_operation(txn, &request, existing).await; + } + if epoch != WRITER_EPOCH || sequence < 0 { + return Err(PublicationReceiptError::Conflict( + "writer epoch is fenced".into(), + )); + } + reject_legacy_queue_receipt( + txn, + request.legacy_operation_id.as_deref(), + &request.namespace, + ) + .await?; + Ok(PublicationPreparation::Prepared(PreparedPublication { + request, + sequence, + transaction_id: row.try_get("", "transaction_id")?, + })) + } + + /// Finalize only the request reserved before this transaction's business writes. + pub(crate) async fn record_publication_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedPublication, + old_oid: &str, + new_oid: &str, + ) -> Result { + let request = prepared.request; + if request.is_noop { + return Err(PublicationReceiptError::Integrity( + "no-op requests do not publish".into(), + )); + } + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE mst2_namespace_seq SET sequence = sequence + 1 \ + WHERE namespace = $1 AND sequence = $2 AND epoch = $3 AND txid_current() = $4 \ + RETURNING sequence", + [ + request.namespace.clone().into(), + prepared.sequence.into(), + WRITER_EPOCH.into(), + prepared.transaction_id.into(), + ], + )) + .await? + .ok_or_else(|| { + PublicationReceiptError::Conflict( + "publication reservation is stale or belongs to another transaction".into(), + ) + })?; + let sequence: i64 = row.try_get("", "sequence")?; + let now = chrono::Utc::now().fixed_offset(); + let receipt = mst2_publication::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id.clone()), + namespace: Set(request.namespace.clone()), + sequence: Set(sequence), + old_oid: Set(old_oid.to_owned()), + new_oid: Set(new_oid.to_owned()), + writer_epoch: Set(WRITER_EPOCH), + writer_kind: Set(request.writer_kind), + request_digest: Set(Some(request.request_digest)), + request_digest_version: Set(Some(REQUEST_DIGEST_VERSION)), + native_certificate_version: Set(None), + created_at: Set(now), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("receipt-written"); + let outbox = mst2_publication_outbox::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id), + namespace: Set(request.namespace), + sequence: Set(sequence), + state: Set("PENDING".to_owned()), + created_at: Set(now), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("outbox-written"); + Ok(CommittedPublication { receipt, outbox }) + } + + pub(crate) async fn record_noop_operation_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedPublication, + root_commit: &str, + root_tree: &str, + landed_commit_id: &str, + ) -> Result { + if !prepared.request.is_noop { + return Err(PublicationReceiptError::Integrity( + "request is not a trunk push no-op".into(), + )); + } + txn.query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT sequence FROM mst2_namespace_seq \ + WHERE namespace = $1 AND sequence = $2 AND epoch = $3 AND txid_current() = $4 FOR UPDATE", + [prepared.request.namespace.clone().into(), prepared.sequence.into(), WRITER_EPOCH.into(), prepared.transaction_id.into()], + )).await?.ok_or_else(|| PublicationReceiptError::Conflict("no-op reservation is stale or belongs to another transaction".into()))?; + let request = prepared.request; + let receipt = mst2_queue_noop_receipt::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + operation_id: Set(request.operation_id), + namespace: Set(request.namespace), + request_digest: Set(request.request_digest), + request_digest_version: Set(REQUEST_DIGEST_VERSION), + writer_epoch: Set(WRITER_EPOCH), + writer_kind: Set(request.writer_kind), + observed_sequence: Set(prepared.sequence), + observed_root_commit: Set(root_commit.to_owned()), + observed_root_tree: Set(root_tree.to_owned()), + landed_commit_id: Set(landed_commit_id.to_owned()), + created_at: Set(chrono::Utc::now().fixed_offset()), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("noop-receipt-written"); + Ok(receipt) + } + + #[cfg(test)] + pub(crate) async fn record_test_publication_in_txn( + &self, + txn: &DatabaseTransaction, + operation_id: &str, + namespace: &str, + old: &str, + new: &str, + writer_kind: &str, + ) -> Result { + let request = PublicationRequest::for_test(operation_id, namespace, old, new, writer_kind); + let committed = match self.begin_publication_in_txn(txn, request).await? { + PublicationPreparation::AlreadyCommitted(committed) => committed, + PublicationPreparation::AlreadyCommittedNoop(_) => { + return Err(PublicationReceiptError::Integrity( + "test publication replayed a no-op".into(), + )); + } + PublicationPreparation::Prepared(prepared) => { + self.record_publication_in_txn(txn, prepared, old, new) + .await? + } + }; + Ok(committed.receipt.sequence) + } +} + +#[cfg(test)] +#[path = "mst2_publication_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/mst2_publication_tests.rs b/src/jupiter/storage/mst2_publication_tests.rs new file mode 100644 index 00000000..fbc809aa --- /dev/null +++ b/src/jupiter/storage/mst2_publication_tests.rs @@ -0,0 +1,1083 @@ +use std::sync::Arc; + +use sea_orm::{PaginatorTrait, TransactionTrait}; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::jupiter::{ + migration::{Migrator, apply_migrations}, + storage::{ + base_storage::{BaseStorage, StorageConnector}, + push_queue_storage::{ClaimOutcome, EnqueueOutcome, EnqueueParams, PushQueueStorage}, + }, + tests::test_db_connection, +}; + +fn queue_row() -> push_queue::Model { + let now = chrono::Utc::now().fixed_offset(); + push_queue::Model { + id: 7, + kind: PushQueueKindEnum::Push, + operation_id: "old→new".to_owned(), + path: "/project".to_owned(), + old_id: "old".to_owned(), + new_id: "new".to_owned(), + payload: serde_json::json!({"commits": ["new"], "n": 1, "fork_base": "old"}), + landed_commit_id: None, + status: crate::callisto::sea_orm_active_enums::PushQueueStatusEnum::Running, + requester: Some("alice".to_owned()), + failure_type: None, + error_message: None, + heartbeat_at: now, + superseded_by: None, + expected_commit_hash: Some("root-before".to_owned()), + expected_tree_hash: Some("tree-before".to_owned()), + expected_native_sequence: None, + expected_native_epoch: None, + expected_native_certificate: None, + pending_action: None, + enqueued_at: now, + started_at: Some(now), + finished_at: None, + updated_at: now, + } +} + +fn noop_queue_row() -> push_queue::Model { + let mut row = queue_row(); + row.old_id = "tip".to_owned(); + row.new_id = "tip".to_owned(); + row.payload = serde_json::json!({"commits": [], "fork_base": null, "n": 0}); + row +} + +#[test] +fn queue_digest_is_versioned_and_binds_trusted_intent_not_retry_baseline() { + let row = queue_row(); + let original = PublicationRequest::from_trunk_queue(&row).unwrap(); + assert_eq!(original.operation_id, "mst2:trunk-queue:7"); + let mut retry = row.clone(); + retry.expected_commit_hash = Some("later-root".to_owned()); + retry.expected_tree_hash = Some("later-tree".to_owned()); + retry.started_at = None; + retry.payload = serde_json::from_str(r#"{"n":1,"fork_base":"old","commits":["new"]}"#).unwrap(); + assert_eq!( + PublicationRequest::from_trunk_queue(&retry) + .unwrap() + .request_digest, + original.request_digest + ); + for change in ["actor", "path", "old", "new", "payload", "identity"] { + let mut changed = row.clone(); + match change { + "actor" => changed.requester = Some("bob".to_owned()), + "path" => changed.path = "/other".to_owned(), + "old" => changed.old_id = "other-old".to_owned(), + "new" => changed.new_id = "other-new".to_owned(), + "payload" => changed.payload["commits"] = serde_json::json!(["new", "extra"]), + "identity" => changed.id += 1, + _ => unreachable!(), + } + assert_ne!( + PublicationRequest::from_trunk_queue(&changed) + .unwrap() + .request_digest, + original.request_digest, + "{change}" + ); + } +} + +async fn storage() -> (tempfile::TempDir, MonoStorage) { + let temp = tempfile::tempdir().unwrap(); + let db = test_db_connection(temp.path()).await; + apply_migrations(&db, false).await.unwrap(); + ( + temp, + MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }, + ) +} + +async fn seed_root(mono: &MonoStorage) { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mega_refs (id, path, ref_name, ref_commit_hash, ref_tree_hash, created_at, updated_at, is_cl) \ + VALUES (1, '/', $1, $2, $3, now(), now(), false)", + [MEGA_BRANCH_NAME.into(), "a".repeat(40).into(), "b".repeat(40).into()], + )).await.unwrap(); +} + +async fn prepared( + mono: &MonoStorage, + txn: &DatabaseTransaction, + request: PublicationRequest, +) -> PreparedPublication { + match mono.begin_publication_in_txn(txn, request).await.unwrap() { + PublicationPreparation::Prepared(prepared) => prepared, + PublicationPreparation::AlreadyCommitted(_) => panic!("fresh operation"), + PublicationPreparation::AlreadyCommittedNoop(_) => panic!("fresh no-op operation"), + } +} + +async fn advance(mono: &MonoStorage, txn: &DatabaseTransaction) -> bool { + mono.cas_update_root_main_ref_in_txn( + txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"c".repeat(40), + &"d".repeat(40), + ) + .await + .unwrap() +} + +async fn counts(mono: &MonoStorage) -> (u64, u64) { + ( + mst2_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + ) +} + +#[tokio::test] +async fn merge_stub_admission_replays_same_receipt_and_rejects_changed_intent() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let queue = PushQueueStorage::new(mono.base.clone()); + let payload = serde_json::json!({"steps": ["first"], "mode": "stub"}); + let admitted = queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("alice"), + payload: payload.clone(), + }) + .await + .unwrap(); + let EnqueueOutcome::Inserted { id } = admitted else { + panic!("{admitted:?}"); + }; + // With no committed receipt, admission retains the existing adoption + // behavior and the original persisted intent remains the execution input. + assert_eq!( + queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("legacy-retry"), + payload: serde_json::json!({"different": "no receipt yet"}), + }) + .await + .unwrap(), + EnqueueOutcome::Adopted { id } + ); + assert_eq!( + queue.claim_for_execution(id).await.unwrap(), + ClaimOutcome::Claimed + ); + let row = push_queue::Entity::find_by_id(id) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(row.requester.as_deref(), Some("alice")); + assert_eq!(row.payload, payload); + let txn = mono.get_connection().begin().await.unwrap(); + let reservation = prepared( + &mono, + &txn, + PublicationRequest::from_trunk_queue(&row).unwrap(), + ) + .await; + assert!(advance(&mono, &txn).await); + let landed = "c".repeat(40); + assert!( + PushQueueStorage::mark_done_if_running_in_txn(&txn, id, &landed) + .await + .unwrap() + ); + let original = mono + .record_publication_in_txn(&txn, reservation, &"a".repeat(40), &landed) + .await + .unwrap(); + assert_eq!(original.receipt.writer_kind, "trunk_merge_stub"); + txn.commit().await.unwrap(); + assert_eq!( + queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some("alice"), + payload: payload.clone(), + }) + .await + .unwrap(), + EnqueueOutcome::Replay { + id, + landed_commit_id: Some(landed.clone()) + } + ); + for change in ["actor", "payload"] { + let (requester, retry_payload) = if change == "actor" { + ("mallory", payload.clone()) + } else { + ( + "alice", + serde_json::json!({"steps": ["first", "changed"], "mode": "stub"}), + ) + }; + let error = queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Merge, + operation_id: "merge-stub-admission", + path: "/project", + old_id: "old", + new_id: "new", + requester: Some(requester), + payload: retry_payload, + }) + .await + .unwrap_err(); + assert!( + error.to_string().contains("MST2_PUBLICATION_CONFLICT"), + "{change}" + ); + } + assert_eq!( + mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + vec![original.receipt] + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + vec![original.outbox] + ); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 1); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + landed + ); +} + +#[tokio::test] +async fn same_request_concurrently_returns_one_original_receipt_and_outbox() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let request = PublicationRequest::for_test("same-op", "/", "a", "c", "test"); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for _ in 0..2 { + let mono = mono.clone(); + let request = request.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let (committed, wrote) = + match mono.begin_publication_in_txn(&txn, request).await.unwrap() { + PublicationPreparation::AlreadyCommitted(committed) => (committed, false), + PublicationPreparation::AlreadyCommittedNoop(_) => { + panic!("publication request") + } + PublicationPreparation::Prepared(prepared) => { + assert!(advance(&mono, &txn).await); + ( + mono.record_publication_in_txn(&txn, prepared, "a", "c") + .await + .unwrap(), + true, + ) + } + }; + txn.commit().await.unwrap(); + (committed, wrote) + })); + } + let mut results = Vec::new(); + for worker in workers { + results.push( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(results.iter().filter(|(_, wrote)| *wrote).count(), 1); + assert_eq!(results[0].0.receipt, results[1].0.receipt); + assert_eq!(results[0].0.outbox, results[1].0.outbox); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 1); + assert_eq!(counts(&mono).await, (1, 1)); +} + +#[tokio::test] +async fn changed_request_conflicts_before_ref_writes() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let txn = mono.get_connection().begin().await.unwrap(); + let request = PublicationRequest::from_trunk_queue(&queue_row()).unwrap(); + let reserved = prepared(&mono, &txn, request).await; + assert!(advance(&mono, &txn).await); + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + for change in ["actor", "new", "payload"] { + let mut changed = queue_row(); + match change { + "actor" => changed.requester = Some("mallory".to_owned()), + "new" => changed.new_id = "overwrite".to_owned(), + "payload" => changed.payload["commits"] = serde_json::json!(["injected"]), + _ => unreachable!(), + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&changed).unwrap() + ) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "c".repeat(40) + ); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 1); + assert_eq!(counts(&mono).await, (1, 1)); + } +} + +#[tokio::test] +async fn different_operations_compete_on_actual_root_cas_without_extra_sequence() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for i in 0..2 { + let mono = mono.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let request = + PublicationRequest::for_test(&format!("competitor-{i}"), "/", "a", "c", "test"); + let reserved = prepared(&mono, &txn, request).await; + if !advance(&mono, &txn).await { + txn.rollback().await.unwrap(); + return false; + } + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + true + })); + } + let mut winners = 0; + for worker in workers { + winners += usize::from( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(winners, 1); + assert_eq!(counts(&mono).await, (1, 1)); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 1); +} + +#[tokio::test] +async fn reservation_cannot_be_finalized_from_another_transaction() { + let (_temp, mono) = storage().await; + // Keep the namespace/sequence unchanged after rollback: only the txn id differs. + mono.get_connection() + .execute_unprepared( + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ('/', 0, 1)", + ) + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared( + &mono, + &txn, + PublicationRequest::for_test("txn-op", "/", "a", "c", "test"), + ) + .await; + txn.rollback().await.unwrap(); + let other = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.record_publication_in_txn(&other, reserved, "a", "c") + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + other.rollback().await.unwrap(); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/").await.unwrap(), 0); +} + +#[tokio::test] +async fn legacy_receipt_and_queue_alias_do_not_guess_request_digest() { + let (_temp, mono) = storage().await; + let row = queue_row(); + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ($1, $2, 1, 'old', 'new', 1, 'trunk_push', now())", + [row.operation_id.clone().into(), row.path.clone().into()], + )).await.unwrap(); + for request in [ + PublicationRequest::for_test(&row.operation_id, &row.path, "old", "new", "trunk_push"), + PublicationRequest::from_trunk_queue(&row).unwrap(), + ] { + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn(&txn, request).await, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + txn.rollback().await.unwrap(); + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + txn.rollback().await.unwrap(); + let legacy = mst2_publication::Entity::find() + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); + assert!(legacy.request_digest_version.is_none()); + assert_eq!(counts(&mono).await, (1, 0)); +} + +#[tokio::test] +async fn incomplete_outbox_refuses_both_receipt_and_queue_replay() { + for corruption in ["missing", "sequence", "namespace"] { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let row = queue_row(); + let request = PublicationRequest::from_trunk_queue(&row).unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared(&mono, &txn, request.clone()).await; + assert!(advance(&mono, &txn).await); + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + txn.commit().await.unwrap(); + let sql = match corruption { + "missing" => "DELETE FROM mst2_publication_outbox", + "sequence" => "UPDATE mst2_publication_outbox SET sequence = sequence + 1", + "namespace" => "UPDATE mst2_publication_outbox SET namespace = '/other'", + _ => unreachable!(), + }; + mono.get_connection().execute_unprepared(sql).await.unwrap(); + let before = counts(&mono).await; + let txn = mono.get_connection().begin().await.unwrap(); + assert!( + matches!( + mono.begin_publication_in_txn(&txn, request).await, + Err(PublicationReceiptError::Integrity(_)) + ), + "{corruption}" + ); + txn.rollback().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!( + matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::Integrity(_)) + ), + "{corruption}" + ); + txn.rollback().await.unwrap(); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "c".repeat(40) + ); + assert_eq!(mono.publication_sequence(&row.path).await.unwrap(), 1); + assert_eq!(counts(&mono).await, before); + } +} + +#[tokio::test] +async fn concurrent_noop_operations_keep_the_publication_sequence_and_outbox() { + let (_temp, mono) = storage().await; + seed_root(&mono).await; + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let mut workers = Vec::new(); + for _ in 0..2 { + let mono = mono.clone(); + let barrier = Arc::clone(&barrier); + workers.push(tokio::spawn(async move { + let txn = mono.get_connection().begin().await.unwrap(); + barrier.wait().await; + let (receipt, wrote) = match mono + .begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await + .unwrap() + { + PublicationPreparation::Prepared(reserved) => { + assert!( + mono.cas_update_root_main_ref_in_txn( + &txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"a".repeat(40), + &"b".repeat(40) + ) + .await + .unwrap() + ); + ( + mono.record_noop_operation_in_txn( + &txn, + reserved, + &"a".repeat(40), + &"b".repeat(40), + "tip", + ) + .await + .unwrap(), + true, + ) + } + PublicationPreparation::AlreadyCommittedNoop(receipt) => (receipt, false), + PublicationPreparation::AlreadyCommitted(_) => panic!("no-op request"), + }; + txn.commit().await.unwrap(); + (receipt, wrote) + })); + } + let mut results = Vec::new(); + for worker in workers { + results.push( + tokio::time::timeout(std::time::Duration::from_secs(30), worker) + .await + .unwrap() + .unwrap(), + ); + } + assert_eq!(results.iter().filter(|(_, wrote)| *wrote).count(), 1); + assert_eq!(results[0].0, results[1].0); + assert_eq!(results[0].0.observed_sequence, 0); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + for change in ["actor", "payload"] { + let mut row = noop_queue_row(); + if change == "actor" { + row.requester = Some("other".to_owned()); + } else { + row.payload["fork_base"] = serde_json::json!("other"); + } + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&row).unwrap() + ) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + } +} + +#[tokio::test] +async fn noop_receipt_replay_rejects_unsupported_version_and_publication_corruption() { + for corruption in ["version", "publication", "outbox"] { + let (_temp, mono) = storage().await; + let row = noop_queue_row(); + let request = PublicationRequest::from_trunk_queue(&row).unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared(&mono, &txn, request.clone()).await; + mono.record_noop_operation_in_txn(&txn, reserved, "root", "tree", "tip") + .await + .unwrap(); + txn.commit().await.unwrap(); + let sql = match corruption { + "version" => "UPDATE mst2_queue_noop_receipt SET request_digest_version = 2", + "publication" => { + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ('mst2:trunk-queue:7', '/project', 1, 'a', 'b', 1, 'trunk_push', now())" + } + "outbox" => { + "INSERT INTO mst2_publication_outbox (operation_id, namespace, sequence, state, created_at) \ + VALUES ('mst2:trunk-queue:7', '/project', 1, 'PENDING', now())" + } + _ => unreachable!(), + }; + mono.get_connection().execute_unprepared(sql).await.unwrap(); + let publications_before = mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(); + let outbox_before = mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(); + for admission in [false, true] { + let txn = mono.get_connection().begin().await.unwrap(); + let result = if admission { + mono.validate_queue_publication_replay_in_txn(&txn, &row) + .await + } else { + mono.begin_publication_in_txn(&txn, request.clone()) + .await + .map(|_| ()) + }; + if corruption == "version" { + assert!(matches!( + result, + Err(PublicationReceiptError::LegacyReceipt(_)) + )); + } else { + assert!(matches!(result, Err(PublicationReceiptError::Integrity(_)))); + } + txn.rollback().await.unwrap(); + let namespace = mono + .get_connection() + .query_one_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "SELECT sequence FROM mst2_namespace_seq WHERE namespace = $1", + [row.path.clone().into()], + )) + .await + .unwrap() + .unwrap(); + assert_eq!(namespace.try_get::("", "sequence").unwrap(), 0); + assert_eq!( + mst2_publication::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + publications_before + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .all(mono.get_connection()) + .await + .unwrap(), + outbox_before + ); + } + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 1 + ); + } +} + +#[tokio::test] +async fn noop_reservation_cannot_move_to_another_transaction() { + let (_temp, mono) = storage().await; + mono.get_connection() + .execute_unprepared( + "INSERT INTO mst2_namespace_seq (namespace, sequence, epoch) VALUES ('/project', 0, 1)", + ) + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let reserved = prepared( + &mono, + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await; + txn.rollback().await.unwrap(); + let other = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.record_noop_operation_in_txn(&other, reserved, "root", "tree", "tip") + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + other.rollback().await.unwrap(); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn migration_preserves_legacy_receipts_and_validates_digest_columns() { + let temp = tempfile::tempdir().unwrap(); + let db = test_db_connection(temp.path()).await; + let old_count = u32::try_from(Migrator::migrations().len() - 2).unwrap(); + Migrator::up(&db, Some(old_count)).await.unwrap(); + db.execute_unprepared( + "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ + VALUES ('legacy', '/', 1, 'a', 'b', 1, 'test', now())" + ).await.unwrap(); + apply_migrations(&db, false).await.unwrap(); + let latest = Migrator::migrations().pop().unwrap(); + latest + .up(&sea_orm_migration::SchemaManager::new(&db)) + .await + .unwrap(); + let legacy = mst2_publication::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); + assert!(legacy.request_digest_version.is_none()); + for (digest, version) in [ + (Some(format!("sha256:{}", "a".repeat(64))), Some(1)), + (Some(format!("sha256:{}", "A".repeat(64))), Some(1)), + (Some("broken".to_owned()), Some(1)), + (None, Some(1)), + (Some(format!("sha256:{}", "a".repeat(64))), None), + (Some(format!("sha256:{}", "a".repeat(64))), Some(0)), + ] { + let valid = digest.as_deref() == Some(format!("sha256:{}", "a".repeat(64)).as_str()) + && version == Some(1); + let txn = db.begin().await.unwrap(); + let result = txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE mst2_publication SET request_digest = $1, request_digest_version = $2 WHERE operation_id = 'legacy'", + [digest.into(), version.into()], + )).await; + assert_eq!(result.is_ok(), valid); + txn.rollback().await.unwrap(); + } + let legacy = mst2_publication::Entity::find() + .one(&db) + .await + .unwrap() + .unwrap(); + assert!(legacy.request_digest.is_none()); +} + +#[cfg(unix)] +pub(super) fn crash_checkpoint(phase: &str) { + if std::env::var("MEGA_MST2_RECEIPT_CRASH_PHASE") + .ok() + .as_deref() + == Some(phase) + { + unsafe { + libc::raise(libc::SIGKILL); + } + panic!("SIGKILL did not terminate the worker"); + } +} + +#[cfg(unix)] +#[test] +fn receipt_crash_worker() { + let Ok(url) = std::env::var("MEGA_MST2_RECEIPT_WORKER_DB") else { + return; + }; + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let db = sea_orm::Database::connect(url).await.unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + let txn = mono.get_connection().begin().await.unwrap(); + let noop = std::env::var("MEGA_MST2_RECEIPT_WORKER_NOOP").is_ok(); + let request = if noop { + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap() + } else { + PublicationRequest::for_test("process-op", "/", "a", "c", "test") + }; + let reserved = prepared(&mono, &txn, request).await; + if noop { + assert!( + mono.cas_update_root_main_ref_in_txn( + &txn, + Some(&"a".repeat(40)), + Some(&"b".repeat(40)), + &"a".repeat(40), + &"b".repeat(40) + ) + .await + .unwrap() + ); + } else { + assert!(advance(&mono, &txn).await); + } + crash_checkpoint("ref-written"); + if noop { + mono.record_noop_operation_in_txn( + &txn, + reserved, + &"a".repeat(40), + &"b".repeat(40), + "tip", + ) + .await + .unwrap(); + } else { + mono.record_publication_in_txn(&txn, reserved, "a", "c") + .await + .unwrap(); + } + txn.commit().await.unwrap(); + crash_checkpoint("commit-complete"); + }); +} + +#[cfg(unix)] +#[tokio::test] +async fn sigkill_recovery_and_commit_without_response_use_original_receipt() { + use std::os::unix::process::ExitStatusExt; + + for phase in [ + "ref-written", + "receipt-written", + "outbox-written", + "commit-complete", + ] { + let temp = tempfile::tempdir().unwrap(); + let (config, _schema) = crate::jupiter::tests::test_db_config(temp.path()).await; + let db = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + seed_root(&mono).await; + let mut worker = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "jupiter::storage::mst2_publication_storage::tests::receipt_crash_worker", + "--nocapture", + ]) + .env("MEGA_MST2_RECEIPT_WORKER_DB", &config.db_url) + .env("MEGA_MST2_RECEIPT_CRASH_PHASE", phase) + .env_remove("MEGA_MST2_RECEIPT_WORKER_NOOP") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let status = loop { + if let Some(status) = worker.try_wait().unwrap() { + break status; + } + if std::time::Instant::now() >= deadline { + worker.kill().unwrap(); + worker.wait().unwrap(); + panic!("receipt worker timed out at {phase}"); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }; + assert_eq!(status.signal(), Some(libc::SIGKILL), "{phase}: {status}"); + let committed = phase == "commit-complete"; + let root = mono.get_main_ref("/").await.unwrap().unwrap(); + assert_eq!( + root.ref_commit_hash, + if committed { "c" } else { "a" }.repeat(40) + ); + assert_eq!(counts(&mono).await, if committed { (1, 1) } else { (0, 0) }); + assert_eq!( + mono.publication_sequence("/").await.unwrap(), + i64::from(committed) + ); + let retry_db = sea_orm::Database::connect(config.db_url.as_str()) + .await + .unwrap(); + let retry_mono = MonoStorage { + base: BaseStorage::new(Arc::new(retry_db)), + }; + let txn = retry_mono.get_connection().begin().await.unwrap(); + let request = PublicationRequest::for_test("process-op", "/", "a", "c", "test"); + match retry_mono + .begin_publication_in_txn(&txn, request) + .await + .unwrap() + { + PublicationPreparation::AlreadyCommitted(original) => { + assert!(committed); + assert_eq!(original.receipt.sequence, 1); + assert_eq!(original.receipt.new_oid, "c"); + assert_eq!(original.outbox.sequence, 1); + txn.commit().await.unwrap(); + } + PublicationPreparation::Prepared(_) => { + assert!(!committed); + txn.rollback().await.unwrap(); + } + PublicationPreparation::AlreadyCommittedNoop(_) => panic!("publication request"), + } + assert_eq!(counts(&mono).await, if committed { (1, 1) } else { (0, 0) }); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn noop_sigkill_and_lost_commit_response_never_create_a_publication() { + use std::os::unix::process::ExitStatusExt; + + for phase in ["ref-written", "noop-receipt-written", "commit-complete"] { + let temp = tempfile::tempdir().unwrap(); + let (config, _schema) = crate::jupiter::tests::test_db_config(temp.path()).await; + let db = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let mono = MonoStorage { + base: BaseStorage::new(Arc::new(db)), + }; + seed_root(&mono).await; + let mut worker = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "jupiter::storage::mst2_publication_storage::tests::receipt_crash_worker", + "--nocapture", + ]) + .env("MEGA_MST2_RECEIPT_WORKER_DB", &config.db_url) + .env("MEGA_MST2_RECEIPT_CRASH_PHASE", phase) + .env("MEGA_MST2_RECEIPT_WORKER_NOOP", "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let status = loop { + if let Some(status) = worker.try_wait().unwrap() { + break status; + } + if std::time::Instant::now() >= deadline { + worker.kill().unwrap(); + worker.wait().unwrap(); + panic!("no-op receipt worker timed out at {phase}"); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }; + assert_eq!(status.signal(), Some(libc::SIGKILL), "{phase}: {status}"); + let committed = phase == "commit-complete"; + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "a".repeat(40) + ); + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!( + mst2_queue_noop_receipt::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + u64::from(committed) + ); + let original = mst2_queue_noop_receipt::Entity::find() + .one(mono.get_connection()) + .await + .unwrap(); + if committed { + let txn = mono.get_connection().begin().await.unwrap(); + assert!(advance(&mono, &txn).await); + txn.commit().await.unwrap(); + } + // Reconnect after the worker died; committed no-op identity survives a later root. + let retry_db = sea_orm::Database::connect(config.db_url.as_str()) + .await + .unwrap(); + let retry_mono = MonoStorage { + base: BaseStorage::new(Arc::new(retry_db)), + }; + let txn = retry_mono.get_connection().begin().await.unwrap(); + match retry_mono + .begin_publication_in_txn( + &txn, + PublicationRequest::from_trunk_queue(&noop_queue_row()).unwrap(), + ) + .await + .unwrap() + { + PublicationPreparation::AlreadyCommittedNoop(receipt) => { + assert!(committed); + assert_eq!(Some(receipt.clone()), original); + assert_eq!(receipt.observed_sequence, 0); + assert_eq!(receipt.observed_root_commit, "a".repeat(40)); + assert_eq!(receipt.landed_commit_id, "tip"); + txn.commit().await.unwrap(); + } + PublicationPreparation::Prepared(_) => { + assert!(!committed); + txn.rollback().await.unwrap(); + } + PublicationPreparation::AlreadyCommitted(_) => panic!("no-op must not publish"), + } + assert_eq!(counts(&mono).await, (0, 0)); + assert_eq!(mono.publication_sequence("/project").await.unwrap(), 0); + assert_eq!( + mono.get_main_ref("/") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + if committed { "c" } else { "a" }.repeat(40) + ); + } +} diff --git a/src/jupiter/storage/native_publication_storage.rs b/src/jupiter/storage/native_publication_storage.rs new file mode 100644 index 00000000..6ccb3fc9 --- /dev/null +++ b/src/jupiter/storage/native_publication_storage.rs @@ -0,0 +1,535 @@ +//! Native `/` publication certificates; origin path receipts retain their v1 identity. + +use git_internal::hash::{ObjectHash, get_hash_kind}; +use sea_orm::{ + ActiveModelTrait, ActiveValue::Set, ColumnTrait, ConnectionTrait, DatabaseTransaction, + EntityTrait, QueryFilter, QueryResult, Statement, TransactionTrait, +}; + +use crate::{ + callisto::{mega_refs, mst2_native_publication, mst2_publication, push_queue}, + common::utils::MEGA_BRANCH_NAME, + jupiter::storage::{ + base_storage::StorageConnector, + mono_storage::MonoStorage, + mst2_publication_storage::{CommittedPublication, PublicationReceiptError}, + push_queue_storage::PushQueueStorage, + }, +}; + +const NATIVE_EPOCH: i64 = 1; + +// Used by both ordinary reads and the single UPDATE that captures a B2.5 token. +pub(crate) const NATIVE_OBSERVATION_CTES: &str = r#" + native_roots AS ( + SELECT count(*)::bigint AS root_count, + max(ref_commit_hash) AS root_commit, max(ref_tree_hash) AS root_tree + FROM mega_refs WHERE path = '/' AND ref_name = $1 AND is_cl = false + ), native_observation AS ( + SELECT roots.*, h.instance_id, h.sequence, h.writer_epoch, h.state, + h.root_commit AS head_commit, h.root_tree AS head_tree, + h.certificate_receipt_id, + c.receipt_id AS certificate_id, c.namespace AS certificate_namespace, + c.instance_id AS certificate_instance, c.sequence AS certificate_sequence, + c.writer_epoch AS certificate_epoch, + c.old_root_commit AS certificate_old_commit, + c.root_commit AS certificate_commit, c.root_tree AS certificate_tree, + c.origin_path, c.origin_ref, c.old_path_commit, c.path_commit, + r.id AS receipt_id, r.namespace AS receipt_namespace, + r.sequence AS receipt_sequence, r.operation_id, + r.old_oid, r.new_oid, r.writer_epoch AS receipt_epoch, r.writer_kind, + r.request_digest, r.request_digest_version, r.native_certificate_version, + o.id AS outbox_id, o.namespace AS outbox_namespace, o.sequence AS outbox_sequence + FROM native_roots roots + LEFT JOIN mst2_native_head h ON h.namespace = '/' + LEFT JOIN mst2_native_publication c ON c.receipt_id = h.certificate_receipt_id + LEFT JOIN mst2_publication r ON r.id = c.receipt_id + LEFT JOIN mst2_publication_outbox o ON o.operation_id = r.operation_id + ) +"#; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct NativeRoot { + pub(crate) commit: String, + pub(crate) tree: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct NativePublicationToken { + pub(crate) sequence: i64, + pub(crate) epoch: i64, + pub(crate) certificate: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct NativePublicationHead { + pub(crate) root: NativeRoot, + pub(crate) instance_id: String, + pub(crate) token: NativePublicationToken, + ready: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct NativeObservation { + pub(crate) root: Option, + pub(crate) head: Option, +} + +#[derive(Debug)] +pub(crate) struct PreparedNativePublication { + head: NativePublicationHead, + path: String, + old_path: Option, + transaction_id: i64, +} + +fn integrity(message: &str) -> PublicationReceiptError { + PublicationReceiptError::Integrity(message.to_owned()) +} + +fn canonical_instance(instance: &str) -> Result { + uuid::Uuid::parse_str(instance) + .map(|id| id.to_string()) + .map_err(|_| integrity("invalid native deployment instance")) +} + +fn validate_root(root: &NativeRoot) -> Result<(), PublicationReceiptError> { + for oid in [&root.commit, &root.tree] { + ObjectHash::from_hex_for_kind(get_hash_kind(), oid) + .map_err(|_| integrity("invalid native object identity"))?; + } + Ok(()) +} + +pub(crate) fn decode_native_observation( + row: &QueryResult, +) -> Result { + let root_count: i64 = row.try_get("", "root_count")?; + if !(0..=1).contains(&root_count) { + return Err(integrity("native root is ambiguous")); + } + let root = if root_count == 1 { + let root = NativeRoot { + commit: row.try_get("", "root_commit")?, + tree: row.try_get("", "root_tree")?, + }; + validate_root(&root)?; + Some(root) + } else { + None + }; + let instance: Option = row.try_get("", "instance_id")?; + let Some(instance_id) = instance else { + return Ok(NativeObservation { root, head: None }); + }; + if canonical_instance(&instance_id)? != instance_id { + return Err(integrity("native head instance is not canonical")); + } + let head_root = NativeRoot { + commit: row.try_get("", "head_commit")?, + tree: row.try_get("", "head_tree")?, + }; + validate_root(&head_root)?; + if root.as_ref() != Some(&head_root) { + return Err(integrity("native root bypassed its publication head")); + } + let token = NativePublicationToken { + sequence: row.try_get("", "sequence")?, + epoch: row.try_get("", "writer_epoch")?, + certificate: row.try_get("", "certificate_receipt_id")?, + }; + if token.sequence < 0 || token.epoch != NATIVE_EPOCH { + return Err(PublicationReceiptError::Conflict( + "native writer epoch is fenced".into(), + )); + } + let state: String = row.try_get("", "state")?; + let ready = match state.as_str() { + "INITIALIZING" if token.certificate.is_none() => false, + "READY" if token.certificate.is_some() => { + let certificate_id: Option = row.try_get("", "certificate_id")?; + let receipt_id: Option = row.try_get("", "receipt_id")?; + let certificate_sequence: Option = row.try_get("", "certificate_sequence")?; + let certificate_epoch: Option = row.try_get("", "certificate_epoch")?; + let certificate_namespace: Option = row.try_get("", "certificate_namespace")?; + let certificate_instance: Option = row.try_get("", "certificate_instance")?; + let certificate_commit: Option = row.try_get("", "certificate_commit")?; + let certificate_tree: Option = row.try_get("", "certificate_tree")?; + let marker: Option = row.try_get("", "native_certificate_version")?; + let version: Option = row.try_get("", "request_digest_version")?; + let digest: Option = row.try_get("", "request_digest")?; + let writer_kind: Option = row.try_get("", "writer_kind")?; + let receipt_epoch: Option = row.try_get("", "receipt_epoch")?; + let origin_path: Option = row.try_get("", "origin_path")?; + let origin_ref: Option = row.try_get("", "origin_ref")?; + let receipt_namespace: Option = row.try_get("", "receipt_namespace")?; + let old_oid: Option = row.try_get("", "old_oid")?; + let old_root: Option = row.try_get("", "certificate_old_commit")?; + let new_oid: Option = row.try_get("", "new_oid")?; + let path_commit: Option = row.try_get("", "path_commit")?; + let old_path: Option = row.try_get("", "old_path_commit")?; + let outbox: Option = row.try_get("", "outbox_id")?; + let outbox_namespace: Option = row.try_get("", "outbox_namespace")?; + let outbox_sequence: Option = row.try_get("", "outbox_sequence")?; + let receipt_sequence: Option = row.try_get("", "receipt_sequence")?; + let digest_valid = digest.as_deref().is_some_and(|value| { + value.len() == 71 + && value.starts_with("sha256:") + && value[7..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }); + if certificate_id != token.certificate + || receipt_id != token.certificate + || certificate_sequence != Some(token.sequence) + || certificate_epoch != Some(token.epoch) + || certificate_namespace.as_deref() != Some("/") + || certificate_instance.as_deref() != Some(instance_id.as_str()) + || certificate_commit.as_deref() != Some(head_root.commit.as_str()) + || certificate_tree.as_deref() != Some(head_root.tree.as_str()) + || marker != Some(1) + || version != Some(1) + || !digest_valid + || writer_kind.as_deref() != Some("trunk_push") + || receipt_epoch != Some(token.epoch) + || origin_ref.as_deref() != Some(MEGA_BRANCH_NAME) + || origin_path.is_none() + || origin_path != receipt_namespace + || new_oid.is_none() + || new_oid != path_commit + || old_oid.is_none() + || old_oid != old_root + || old_path == path_commit + || outbox.is_none() + || outbox_namespace != receipt_namespace + || outbox_sequence != receipt_sequence + { + return Err(integrity( + "native publication association is incomplete or inconsistent", + )); + } + true + } + _ => return Err(integrity("invalid native head state")), + }; + Ok(NativeObservation { + root, + head: Some(NativePublicationHead { + root: head_root, + instance_id, + token, + ready, + }), + }) +} + +async fn observe( + connection: &C, +) -> Result { + let rows = connection + .query_all_raw(Statement::from_sql_and_values( + connection.get_database_backend(), + format!("WITH {NATIVE_OBSERVATION_CTES} SELECT * FROM native_observation"), + [MEGA_BRANCH_NAME.into()], + )) + .await?; + if rows.len() != 1 { + return Err(integrity("ambiguous native publication observation")); + } + decode_native_observation(&rows[0]) +} + +async fn selected_ref( + connection: &C, + path: &str, +) -> Result, PublicationReceiptError> { + let refs = mega_refs::Entity::find() + .filter(mega_refs::Column::Path.eq(path)) + .filter(mega_refs::Column::RefName.eq(MEGA_BRANCH_NAME)) + .filter(mega_refs::Column::IsCl.eq(false)) + .all(connection) + .await?; + match refs.as_slice() { + [] => Ok(None), + [row] => { + let root = NativeRoot { + commit: row.ref_commit_hash.clone(), + tree: row.ref_tree_hash.clone(), + }; + validate_root(&root)?; + Ok(Some(root)) + } + _ => Err(integrity("selected native ref is ambiguous")), + } +} + +impl MonoStorage { + pub(crate) async fn read_native_publication_head( + &self, + instance: &str, + ) -> Result { + let observation = observe(self.get_connection()).await?; + let head = observation + .head + .ok_or_else(|| integrity("native publication is not initialized"))?; + if !head.ready { + return Err(integrity("native publication is not ready")); + } + if head.instance_id != canonical_instance(instance)? { + return Err(integrity("native instance changed")); + } + Ok(head) + } + + /// Maintenance-only, with stopped old writers and a drained queue; never invoked by resolve. + pub(crate) async fn initialize_native_publication( + &self, + instance: &str, + ) -> Result<(), PublicationReceiptError> { + let instance = canonical_instance(instance)?; + let txn = self.get_connection().begin().await?; + PushQueueStorage::acquire_mono_write_lock(&txn) + .await + .map_err(|error| integrity(&error.to_string()))?; + txn.execute_unprepared("SELECT id FROM queue_control WHERE id=1 FOR UPDATE") + .await?; + let observation = observe(&txn).await?; + if observation.head.is_some() { + return Err(integrity("native head already initialized")); + } + let root = observation + .root + .ok_or_else(|| integrity("native root missing"))?; + let row = txn.query_one_raw(Statement::from_string(txn.get_database_backend(), + "SELECT count(*)::bigint AS active FROM push_queue WHERE status IN ('Queued', 'Running')".to_owned())).await? + .ok_or_else(|| integrity("queue observation missing"))?; + if row.try_get::("", "active")? != 0 { + return Err(integrity( + "queue must be drained before native initialization", + )); + } + let row = txn + .query_one_raw(Statement::from_string( + txn.get_database_backend(), + r#" + SELECT min(sequence) AS minimum, max(sequence) AS maximum FROM ( + SELECT sequence FROM mst2_namespace_seq UNION ALL SELECT sequence FROM mst2_publication + UNION ALL SELECT observed_sequence AS sequence FROM mst2_queue_noop_receipt + ) counters + "# + .to_owned(), + )) + .await? + .ok_or_else(|| integrity("native sequence floor missing"))?; + let minimum: Option = row.try_get("", "minimum")?; + let floor = row.try_get::>("", "maximum")?.unwrap_or(0); + if minimum.is_some_and(|value| value < 0) || floor == i64::MAX { + return Err(integrity("native sequence floor is invalid or exhausted")); + } + txn.execute_raw(Statement::from_sql_and_values(txn.get_database_backend(), + "INSERT INTO mst2_native_head (namespace, instance_id, sequence, writer_epoch, root_commit, root_tree, state) \ + VALUES ('/', $1, $2, $3, $4, $5, 'INITIALIZING')", + [instance.into(), floor.into(), NATIVE_EPOCH.into(), root.commit.into(), root.tree.into()], + )).await?; + txn.commit().await?; + Ok(()) + } + + pub(crate) async fn reserve_native_publication_in_txn( + &self, + txn: &DatabaseTransaction, + queue: &push_queue::Model, + instance: &str, + ) -> Result { + let owner = txn.query_one_raw(Statement::from_string(txn.get_database_backend(), + "SELECT txid_current() AS owner FROM mst2_native_head WHERE namespace = '/' FOR UPDATE".to_owned())) + .await?.ok_or_else(|| integrity("native publication is not initialized"))?; + let observation = observe(txn).await?; + let head = observation + .head + .ok_or_else(|| integrity("native publication is not initialized"))?; + if head.instance_id != canonical_instance(instance)? { + return Err(integrity("native instance changed")); + } + if queue.expected_native_sequence != Some(head.token.sequence) + || queue.expected_native_epoch != Some(head.token.epoch) + || queue.expected_native_certificate != head.token.certificate + || queue.expected_commit_hash.as_deref() != Some(head.root.commit.as_str()) + || queue.expected_tree_hash.as_deref() != Some(head.root.tree.as_str()) + { + return Err(PublicationReceiptError::Conflict( + "claimed native publication token is stale or missing".into(), + )); + } + let old_path = selected_ref(txn, &queue.path).await?; + Ok(PreparedNativePublication { + head, + path: queue.path.clone(), + old_path, + transaction_id: owner.try_get("", "owner")?, + }) + } + + async fn check_native_reservation( + &self, + txn: &DatabaseTransaction, + prepared: &PreparedNativePublication, + ) -> Result<(), PublicationReceiptError> { + let row = txn.query_one_raw(Statement::from_sql_and_values(txn.get_database_backend(), + "SELECT sequence FROM mst2_native_head WHERE namespace = '/' AND sequence = $1 AND writer_epoch = $2 \ + AND certificate_receipt_id IS NOT DISTINCT FROM $3 AND root_commit = $4 AND root_tree = $5 \ + AND txid_current() = $6 FOR UPDATE", + [prepared.head.token.sequence.into(), prepared.head.token.epoch.into(), prepared.head.token.certificate.into(), + prepared.head.root.commit.clone().into(), prepared.head.root.tree.clone().into(), prepared.transaction_id.into()], + )).await?; + if row.is_none() { + return Err(PublicationReceiptError::Conflict( + "native reservation is stale or belongs to another transaction".into(), + )); + } + Ok(()) + } + + pub(crate) async fn finish_native_noop_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedNativePublication, + ) -> Result<(), PublicationReceiptError> { + self.check_native_reservation(txn, &prepared).await?; + if selected_ref(txn, "/").await?.as_ref() != Some(&prepared.head.root) + || selected_ref(txn, &prepared.path).await? != prepared.old_path + { + return Err(integrity("native no-op changed selected refs")); + } + Ok(()) + } + + pub(crate) async fn record_native_publication_in_txn( + &self, + txn: &DatabaseTransaction, + prepared: PreparedNativePublication, + committed: &CommittedPublication, + ) -> Result<(), PublicationReceiptError> { + self.check_native_reservation(txn, &prepared).await?; + let root = selected_ref(txn, "/") + .await? + .ok_or_else(|| integrity("published native root missing"))?; + let path = selected_ref(txn, &prepared.path) + .await? + .ok_or_else(|| integrity("published native path missing"))?; + if prepared + .old_path + .as_ref() + .is_some_and(|old| old.commit == path.commit) + || committed.receipt.namespace != prepared.path + || committed.receipt.new_oid != path.commit + || committed.receipt.old_oid != prepared.head.root.commit + || committed.receipt.writer_kind != "trunk_push" + || committed.receipt.writer_epoch != NATIVE_EPOCH + || committed.receipt.native_certificate_version.is_some() + || committed.outbox.operation_id != committed.receipt.operation_id + || committed.outbox.namespace != committed.receipt.namespace + || committed.outbox.sequence != committed.receipt.sequence + { + return Err(integrity( + "operation did not change its selected native ref", + )); + } + let next = prepared + .head + .token + .sequence + .checked_add(1) + .ok_or_else(|| integrity("native sequence exhausted"))?; + let marked = txn.execute_raw(Statement::from_sql_and_values(txn.get_database_backend(), + "UPDATE mst2_publication SET native_certificate_version = 1 WHERE id = $1 AND native_certificate_version IS NULL", + [committed.receipt.id.into()], + )).await?; + if marked.rows_affected() != 1 { + return Err(integrity("native receipt marker changed")); + } + mst2_native_publication::ActiveModel { + receipt_id: Set(committed.receipt.id), + namespace: Set("/".to_owned()), + instance_id: Set(prepared.head.instance_id), + sequence: Set(next), + writer_epoch: Set(NATIVE_EPOCH), + old_root_commit: Set(prepared.head.root.commit.clone()), + old_root_tree: Set(prepared.head.root.tree.clone()), + root_commit: Set(root.commit.clone()), + root_tree: Set(root.tree.clone()), + origin_path: Set(prepared.path), + origin_ref: Set(MEGA_BRANCH_NAME.to_owned()), + old_path_commit: Set(prepared.old_path.as_ref().map(|old| old.commit.clone())), + old_path_tree: Set(prepared.old_path.map(|old| old.tree)), + path_commit: Set(path.commit), + path_tree: Set(path.tree), + } + .insert(txn) + .await?; + #[cfg(all(test, unix))] + tests::crash_checkpoint("native-certificate-written"); + let updated = txn.execute_raw(Statement::from_sql_and_values(txn.get_database_backend(), + "UPDATE mst2_native_head SET sequence = $1, root_commit = $2, root_tree = $3, state = 'READY', certificate_receipt_id = $4 \ + WHERE namespace = '/' AND sequence = $5 AND writer_epoch = $6 \ + AND certificate_receipt_id IS NOT DISTINCT FROM $7 AND root_commit = $8 AND root_tree = $9 AND txid_current() = $10", + [next.into(), root.commit.into(), root.tree.into(), committed.receipt.id.into(), prepared.head.token.sequence.into(), + NATIVE_EPOCH.into(), prepared.head.token.certificate.into(), prepared.head.root.commit.into(), + prepared.head.root.tree.into(), prepared.transaction_id.into()], + )).await?; + if updated.rows_affected() != 1 { + return Err(PublicationReceiptError::Conflict( + "native head CAS failed".into(), + )); + } + #[cfg(all(test, unix))] + tests::crash_checkpoint("native-head-written"); + Ok(()) + } + + pub(crate) async fn validate_native_historical_receipt_in_txn( + &self, + txn: &DatabaseTransaction, + receipt: &mst2_publication::Model, + ) -> Result<(), PublicationReceiptError> { + match receipt.native_certificate_version { + None => return Ok(()), + Some(1) => {} + _ => return Err(integrity("unsupported native certificate version")), + } + let certificate = mst2_native_publication::Entity::find_by_id(receipt.id) + .one(txn) + .await? + .ok_or_else(|| integrity("historical native certificate missing"))?; + if certificate.namespace != "/" + || certificate.sequence <= 0 + || certificate.writer_epoch != receipt.writer_epoch + || certificate.origin_path != receipt.namespace + || certificate.origin_ref != MEGA_BRANCH_NAME + || receipt.writer_kind != "trunk_push" + || certificate.old_root_commit != receipt.old_oid + || certificate.path_commit != receipt.new_oid + || certificate.old_path_commit.as_ref() == Some(&certificate.path_commit) + || canonical_instance(&certificate.instance_id)? != certificate.instance_id + { + return Err(integrity("historical native certificate identity mismatch")); + } + validate_root(&NativeRoot { + commit: certificate.root_commit, + tree: certificate.root_tree, + })?; + validate_root(&NativeRoot { + commit: certificate.old_root_commit, + tree: certificate.old_root_tree, + })?; + validate_root(&NativeRoot { + commit: certificate.path_commit, + tree: certificate.path_tree, + })?; + Ok(()) + } +} + +#[cfg(test)] +#[path = "native_publication_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/native_publication_tests.rs b/src/jupiter/storage/native_publication_tests.rs new file mode 100644 index 00000000..d7eeaa78 --- /dev/null +++ b/src/jupiter/storage/native_publication_tests.rs @@ -0,0 +1,378 @@ +use std::sync::Arc; + +use sea_orm::{DatabaseTransaction, PaginatorTrait}; + +use super::*; +use crate::{ + callisto::{ + mst2_native_head, mst2_publication_outbox, sea_orm_active_enums::PushQueueKindEnum, + }, + jupiter::{ + migration::apply_migrations, + storage::{ + base_storage::BaseStorage, + mst2_publication_storage::{PublicationPreparation, PublicationRequest}, + push_queue_storage::{ClaimOutcome, EnqueueOutcome, EnqueueParams}, + }, + tests::test_db_connection, + }, +}; + +const INSTANCE: &str = "6ab219b0-4275-45ba-9d7b-7b0b633018cd"; + +async fn fixture() -> (tempfile::TempDir, MonoStorage, PushQueueStorage) { + let temp = tempfile::tempdir().unwrap(); + let db = test_db_connection(temp.path()).await; + apply_migrations(&db, false).await.unwrap(); + let base = BaseStorage::new(Arc::new(db)); + let mono = MonoStorage { base: base.clone() }; + for (id, path, commit, tree) in [(1_i64, "/", 'a', 'b'), (2, "/project", 'c', 'd')] { + mono.get_connection().execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "INSERT INTO mega_refs (id,path,ref_name,ref_commit_hash,ref_tree_hash,created_at,updated_at,is_cl) \ + VALUES ($1,$2,$3,$4,$5,now(),now(),false)", + [id.into(), path.into(), MEGA_BRANCH_NAME.into(), commit.to_string().repeat(40).into(), tree.to_string().repeat(40).into()], + )).await.unwrap(); + } + mono.initialize_native_publication(INSTANCE).await.unwrap(); + ( + temp, + mono, + PushQueueStorage::new(base).with_native_publication(true), + ) +} + +async fn enqueue_claim( + queue: &PushQueueStorage, + old: &str, + new: &str, + n: u64, +) -> push_queue::Model { + let operation = format!("{old}->{new}"); + let inserted = queue + .enqueue_atomic(EnqueueParams { + kind: PushQueueKindEnum::Push, + operation_id: &operation, + path: "/project", + old_id: old, + new_id: new, + requester: Some("alice"), + payload: serde_json::json!({"n":n,"commits":[new]}), + }) + .await + .unwrap(); + let EnqueueOutcome::Inserted { id } = inserted else { + panic!("fresh operation"); + }; + assert_eq!( + queue.claim_for_execution(id).await.unwrap(), + ClaimOutcome::Claimed + ); + queue.get_by_id(id).await.unwrap().unwrap() +} + +async fn publish_same_root( + mono: &MonoStorage, + txn: &DatabaseTransaction, + row: &push_queue::Model, +) -> i64 { + let request = PublicationRequest::from_trunk_queue(row).unwrap(); + let PublicationPreparation::Prepared(origin) = + mono.begin_publication_in_txn(txn, request).await.unwrap() + else { + panic!("fresh publication"); + }; + let native = mono + .reserve_native_publication_in_txn(txn, row, INSTANCE) + .await + .unwrap(); + assert!( + mono.cas_update_root_main_ref_in_txn( + txn, + row.expected_commit_hash.as_deref(), + row.expected_tree_hash.as_deref(), + row.expected_commit_hash.as_deref().unwrap(), + row.expected_tree_hash.as_deref().unwrap() + ) + .await + .unwrap() + ); + txn.execute_raw(Statement::from_sql_and_values(txn.get_database_backend(), + "UPDATE mega_refs SET ref_commit_hash = $1 WHERE path='/project' AND ref_name=$2 AND is_cl=false", + [row.new_id.clone().into(), MEGA_BRANCH_NAME.into()], + )).await.unwrap(); + let committed = mono + .record_publication_in_txn( + txn, + origin, + row.expected_commit_hash.as_deref().unwrap(), + &row.new_id, + ) + .await + .unwrap(); + mono.record_native_publication_in_txn(txn, native, &committed) + .await + .unwrap(); + txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE push_queue SET status='Done', landed_commit_id=$1 WHERE id=$2", + [row.new_id.clone().into(), row.id.into()], + )) + .await + .unwrap(); + committed.receipt.id +} + +#[tokio::test] +async fn same_root_changes_advance_global_head_and_historical_replay_ignores_latest() { + let (_temp, mono, queue) = fixture().await; + let root_before = mono.get_main_ref("/").await.unwrap().unwrap(); + let first = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + assert_eq!(first.expected_native_sequence, Some(0)); + assert_eq!(first.expected_native_epoch, Some(1)); + assert_eq!(first.expected_native_certificate, None); + let txn = mono.get_connection().begin().await.unwrap(); + let first_receipt = publish_same_root(&mono, &txn, &first).await; + txn.commit().await.unwrap(); + let first_head = mono.read_native_publication_head(INSTANCE).await.unwrap(); + assert_eq!(first_head.token.sequence, 1); + assert_eq!(first_head.root.commit, root_before.ref_commit_hash); + assert_eq!(first_head.root.tree, root_before.ref_tree_hash); + assert_eq!(first_head.token.certificate, Some(first_receipt)); + let second = enqueue_claim(&queue, &first.new_id, &"f".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + publish_same_root(&mono, &txn, &second).await; + txn.commit().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let replay = mono + .begin_publication_in_txn(&txn, PublicationRequest::from_trunk_queue(&first).unwrap()) + .await + .unwrap(); + assert!( + matches!(replay, PublicationPreparation::AlreadyCommitted(ref committed) if committed.receipt.id == first_receipt) + ); + txn.rollback().await.unwrap(); + assert_eq!( + mono.read_native_publication_head(INSTANCE) + .await + .unwrap() + .token + .sequence, + 2 + ); + assert_eq!( + mst2_native_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 2 + ); +} + +#[tokio::test] +async fn stale_same_root_claim_token_rejects_before_mutation_and_reset_clears_the_group() { + let (_temp, mono, queue) = fixture().await; + let row = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + // Independent transaction advances the head without changing either root OID. + mono.get_connection() + .execute_unprepared("UPDATE mst2_native_head SET sequence=1 WHERE namespace='/'") + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.reserve_native_publication_in_txn(&txn, &row, INSTANCE) + .await, + Err(PublicationReceiptError::Conflict(_)) + )); + txn.rollback().await.unwrap(); + assert_eq!( + mono.get_main_ref("/project") + .await + .unwrap() + .unwrap() + .ref_commit_hash, + "c".repeat(40) + ); + assert!(queue.reset_running_to_queued(row.id).await.unwrap()); + let reset = queue.get_by_id(row.id).await.unwrap().unwrap(); + assert_eq!( + ( + reset.expected_commit_hash, + reset.expected_tree_hash, + reset.expected_native_sequence, + reset.expected_native_epoch, + reset.expected_native_certificate + ), + (None, None, None, None, None) + ); +} + +#[tokio::test] +async fn an_uncommitted_certificate_never_exposes_a_partial_head_to_another_connection() { + let (_temp, mono, queue) = fixture().await; + let first = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + publish_same_root(&mono, &txn, &first).await; + txn.commit().await.unwrap(); + let row = enqueue_claim(&queue, &first.new_id, &"f".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + publish_same_root(&mono, &txn, &row).await; + // The pool has two actual connections: one is held by txn, this read is on the other. + let before = tokio::time::timeout( + std::time::Duration::from_secs(5), + mono.read_native_publication_head(INSTANCE), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(before.token.sequence, 1); + txn.commit().await.unwrap(); + let after = mono.read_native_publication_head(INSTANCE).await.unwrap(); + assert_eq!(after.token.sequence, 2); + assert_ne!(before.token.certificate, after.token.certificate); +} + +#[tokio::test] +async fn marked_history_fails_without_certificate_but_literal_v1_history_replays() { + let (_temp, mono, queue) = fixture().await; + let row = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + let receipt_id = publish_same_root(&mono, &txn, &row).await; + txn.commit().await.unwrap(); + mono.get_connection() + .execute_unprepared("DELETE FROM mst2_native_head; DELETE FROM mst2_native_publication;") + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn(&txn, PublicationRequest::from_trunk_queue(&row).unwrap()) + .await, + Err(PublicationReceiptError::Integrity(_)) + )); + txn.rollback().await.unwrap(); + mono.get_connection() + .execute_raw(Statement::from_sql_and_values( + mono.get_connection().get_database_backend(), + "UPDATE mst2_publication SET native_certificate_version=NULL WHERE id=$1", + [receipt_id.into()], + )) + .await + .unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.begin_publication_in_txn(&txn, PublicationRequest::from_trunk_queue(&row).unwrap()) + .await + .unwrap(), + PublicationPreparation::AlreadyCommitted(_) + )); + txn.rollback().await.unwrap(); +} + +#[tokio::test] +async fn initialization_uses_only_a_counter_floor_and_cannot_reinitialize_or_serve_it() { + let (_temp, mono, queue) = fixture().await; + assert!(mono.read_native_publication_head(INSTANCE).await.is_err()); + assert!(mono.initialize_native_publication(INSTANCE).await.is_err()); + mono.get_connection().execute_unprepared("DELETE FROM mst2_native_head; INSERT INTO mst2_namespace_seq(namespace,sequence,epoch) VALUES('/older',19,1)").await.unwrap(); + mono.initialize_native_publication(INSTANCE).await.unwrap(); + let head = mst2_native_head::Entity::find() + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(head.sequence, 19); + assert_eq!(head.state, "INITIALIZING"); + assert_eq!(head.certificate_receipt_id, None); + let row = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + publish_same_root(&mono, &txn, &row).await; + txn.commit().await.unwrap(); + assert_eq!( + mono.read_native_publication_head(INSTANCE) + .await + .unwrap() + .token + .sequence, + 20 + ); +} + +#[tokio::test] +async fn reservation_owner_and_unchanged_selected_ref_cannot_issue_a_certificate() { + let (_temp, mono, queue) = fixture().await; + let row = enqueue_claim(&queue, &"c".repeat(40), &"e".repeat(40), 1).await; + let txn = mono.get_connection().begin().await.unwrap(); + let native = mono + .reserve_native_publication_in_txn(&txn, &row, INSTANCE) + .await + .unwrap(); + txn.rollback().await.unwrap(); + let other = mono.get_connection().begin().await.unwrap(); + assert!(matches!( + mono.finish_native_noop_in_txn(&other, native).await, + Err(PublicationReceiptError::Conflict(_)) + )); + other.rollback().await.unwrap(); + let txn = mono.get_connection().begin().await.unwrap(); + let native = mono + .reserve_native_publication_in_txn(&txn, &row, INSTANCE) + .await + .unwrap(); + let PublicationPreparation::Prepared(origin) = mono + .begin_publication_in_txn(&txn, PublicationRequest::from_trunk_queue(&row).unwrap()) + .await + .unwrap() + else { + panic!("fresh publication"); + }; + let committed = mono + .record_publication_in_txn( + &txn, + origin, + row.expected_commit_hash.as_deref().unwrap(), + &row.old_id, + ) + .await + .unwrap(); + assert!( + mono.record_native_publication_in_txn(&txn, native, &committed) + .await + .is_err() + ); + txn.rollback().await.unwrap(); + assert_eq!( + mst2_native_publication::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_publication_outbox::Entity::find() + .count(mono.get_connection()) + .await + .unwrap(), + 0 + ); +} + +#[cfg(unix)] +pub(super) fn crash_checkpoint(phase: &str) { + if std::env::var("MEGA_MST2_NATIVE_CRASH_PHASE") + .ok() + .as_deref() + == Some(phase) + { + unsafe { + libc::raise(libc::SIGKILL); + } + panic!("SIGKILL did not terminate the native worker"); + } +} diff --git a/src/jupiter/storage/push_queue_storage.rs b/src/jupiter/storage/push_queue_storage.rs index 06878580..8f7616b3 100644 --- a/src/jupiter/storage/push_queue_storage.rs +++ b/src/jupiter/storage/push_queue_storage.rs @@ -101,6 +101,7 @@ pub struct MonoWriteLockHolder { #[derive(Clone)] pub struct PushQueueStorage { base: BaseStorage, + native_publication_enabled: bool, } impl Deref for PushQueueStorage { @@ -123,7 +124,15 @@ pub struct EnqueueParams<'a> { impl PushQueueStorage { pub fn new(base: BaseStorage) -> Self { - Self { base } + Self { + base, + native_publication_enabled: false, + } + } + + pub(crate) fn with_native_publication(mut self, enabled: bool) -> Self { + self.native_publication_enabled = enabled; + self } /// B1: lock `queue_control`, conditional INSERT, classify on zero rows. @@ -241,8 +250,9 @@ impl PushQueueStorage { }); } // operation_states race: winner already committed — classify by read. + let replay_txn = self.get_connection().begin().await?; if let Some(existing) = push_queue::Entity::find() - .filter(push_queue::Column::Kind.eq(kind)) + .filter(push_queue::Column::Kind.eq(kind.clone())) .filter(push_queue::Column::Path.eq(path)) .filter(push_queue::Column::OperationId.eq(operation_id)) .filter(push_queue::Column::Status.is_in([ @@ -250,9 +260,23 @@ impl PushQueueStorage { PushQueueStatusEnum::Running, PushQueueStatusEnum::Done, ])) - .one(self.get_connection()) + .one(&replay_txn) .await? { + if matches!(&kind, PushQueueKindEnum::Push | PushQueueKindEnum::Merge) { + let mut candidate = existing.clone(); + candidate.old_id = old_id.to_owned(); + candidate.new_id = new_id.to_owned(); + candidate.requester = requester.map(str::to_owned); + candidate.payload = payload.clone(); + crate::jupiter::storage::mono_storage::MonoStorage { + base: self.base.clone(), + } + .validate_queue_publication_replay_in_txn(&replay_txn, &candidate) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } + replay_txn.commit().await?; return match existing.status { PushQueueStatusEnum::Done => Ok(EnqueueOutcome::Replay { id: existing.id, @@ -264,6 +288,7 @@ impl PushQueueStorage { _ => unreachable!("filter restricts status"), }; } + replay_txn.rollback().await?; return Err(MegaError::Other(format!( "unique conflict without classifiable row: {msg}" ))); @@ -283,6 +308,29 @@ impl PushQueueStorage { Some(outcome) => { match &outcome { EnqueueOutcome::Adopted { .. } | EnqueueOutcome::Replay { .. } => { + if matches!(&kind, PushQueueKindEnum::Push | PushQueueKindEnum::Merge) { + let id = match &outcome { + EnqueueOutcome::Adopted { id } + | EnqueueOutcome::Replay { id, .. } => *id, + _ => unreachable!("adopt or replay"), + }; + let mut candidate = push_queue::Entity::find_by_id(id) + .one(&txn) + .await? + .ok_or_else(|| { + MegaError::Other("queue replay row missing".into()) + })?; + candidate.old_id = old_id.to_owned(); + candidate.new_id = new_id.to_owned(); + candidate.requester = requester.map(str::to_owned); + candidate.payload = payload.clone(); + crate::jupiter::storage::mono_storage::MonoStorage { + base: self.base.clone(), + } + .validate_queue_publication_replay_in_txn(&txn, &candidate) + .await + .map_err(|error| MegaError::Other(error.to_string()))?; + } txn.commit().await?; } EnqueueOutcome::Rejected { .. } => { @@ -722,66 +770,63 @@ impl PushQueueStorage { return Ok(ClaimOutcome::HardStopped); } - let claimed = txn - .execute_raw(Statement::from_sql_and_values( + // One statement snapshot supplies both the successful claim and its + // root/publication token. Updating the same queue row twice in a + // data-modifying CTE is deliberately avoided. + let sql = format!( + r#" + WITH {native_ctes}, claimed AS ( + UPDATE push_queue q + SET status = 'Running'::push_queue_status_enum, + started_at = now(), heartbeat_at = now(), updated_at = now(), + expected_commit_hash = observation.root_commit, + expected_tree_hash = observation.root_tree, + expected_native_sequence = observation.sequence, + expected_native_epoch = observation.writer_epoch, + expected_native_certificate = observation.certificate_receipt_id + FROM native_observation observation + WHERE q.id = $2 AND q.status = 'Queued'::push_queue_status_enum + AND (SELECT NOT hard_stopped FROM queue_control WHERE id = 1) + AND NOT EXISTS (SELECT 1 FROM push_queue WHERE status = 'Running'::push_queue_status_enum) + AND q.id = (SELECT min(id) FROM push_queue WHERE status = 'Queued'::push_queue_status_enum) + RETURNING q.id + ) + SELECT observation.* FROM native_observation observation + CROSS JOIN claimed + "#, + native_ctes = super::native_publication_storage::NATIVE_OBSERVATION_CTES + .replace("h.namespace = '/'", "h.namespace = '/' AND $3") + ); + let observations = txn + .query_all_raw(Statement::from_sql_and_values( DbBackend::Postgres, - r#" - UPDATE push_queue - SET status = 'Running'::push_queue_status_enum, - started_at = now(), - heartbeat_at = now(), - updated_at = now() - WHERE id = $1 - AND status = 'Queued'::push_queue_status_enum - AND (SELECT NOT hard_stopped FROM queue_control WHERE id = 1) - AND NOT EXISTS ( - SELECT 1 FROM push_queue - WHERE status = 'Running'::push_queue_status_enum - ) - AND id = ( - SELECT min(id) FROM push_queue - WHERE status = 'Queued'::push_queue_status_enum - ) - "#, - [Value::from(id)], + sql, + [ + MEGA_BRANCH_NAME.into(), + id.into(), + self.native_publication_enabled.into(), + ], )) .await?; - - if claimed.rows_affected() == 0 { + if observations.is_empty() { txn.rollback().await?; return Ok(ClaimOutcome::Missed); } - - // Snapshot root AFTER the claim UPDATE succeeds (still in this txn). - // Ordering matters: - // - Before UPDATE: a legitimate predecessor B3 can commit Done + new - // root in the gap → stale expected_* → false QueueBypassDetected. - // - FOR SHARE before UPDATE: blocks admissions while a writer holds - // the root row exclusively (stalls past wait_timeout). - // - After UPDATE: NOT EXISTS(Running) already held, so no legitimate - // B3 can still be writing; plain SELECT needs no row lock. - let root = mega_refs::Entity::find() - .filter(mega_refs::Column::Path.eq("/")) - .filter(mega_refs::Column::RefName.eq(MEGA_BRANCH_NAME.to_owned())) - .one(&txn) - .await?; - let (commit, tree) = match root { - Some(r) => (Some(r.ref_commit_hash), Some(r.ref_tree_hash)), - None => (None, None), - }; - - txn.execute_raw(Statement::from_sql_and_values( - DbBackend::Postgres, - r#" - UPDATE push_queue - SET expected_commit_hash = $2, - expected_tree_hash = $3, - updated_at = now() - WHERE id = $1 - "#, - [Value::from(id), Value::from(commit), Value::from(tree)], - )) - .await?; + if observations.len() != 1 { + return Err(MegaError::Other( + "ambiguous native claim observation".into(), + )); + } + if self.native_publication_enabled { + let observation = + super::native_publication_storage::decode_native_observation(&observations[0]) + .map_err(|error| MegaError::Other(error.to_string()))?; + if observation.head.is_none() { + return Err(MegaError::Other( + "native publication is not initialized".into(), + )); + } + } txn.commit().await?; Ok(ClaimOutcome::Claimed) @@ -899,6 +944,9 @@ impl PushQueueStorage { started_at = NULL, expected_commit_hash = NULL, expected_tree_hash = NULL, + expected_native_sequence = NULL, + expected_native_epoch = NULL, + expected_native_certificate = NULL, heartbeat_at = now(), updated_at = now() WHERE id = $1 @@ -985,6 +1033,9 @@ impl PushQueueStorage { started_at = NULL, expected_commit_hash = NULL, expected_tree_hash = NULL, + expected_native_sequence = NULL, + expected_native_epoch = NULL, + expected_native_certificate = NULL, pending_action = NULL, heartbeat_at = now(), updated_at = now() diff --git a/src/jupiter/tests.rs b/src/jupiter/tests.rs index aab57e64..bd24a106 100644 --- a/src/jupiter/tests.rs +++ b/src/jupiter/tests.rs @@ -357,7 +357,8 @@ pub async fn test_storage_with_config(temp_dir: impl AsRef, config: Config vault_storage: VaultStorage { base: base.clone() }, conversation_storage: ConversationStorage { base: base.clone() }, commit_binding_storage: CommitBindingStorage { base: base.clone() }, - push_queue_storage: PushQueueStorage::new(base.clone()), + push_queue_storage: PushQueueStorage::new(base.clone()) + .with_native_publication(config.mst2.publication_enabled), buck_storage: BuckStorage { base: base.clone() }, bots_storage: BotsStorage { base: base.clone() }, webhook_storage: WebhookStorage { base: base.clone() }, @@ -379,7 +380,8 @@ pub async fn test_storage_with_config(temp_dir: impl AsRef, config: Config config.monorepo.push_policy.clone(), ) .with_timeouts(Duration::from_secs(30), Duration::from_millis(20)) - .with_max_push_commits(config.monorepo.max_push_commits), + .with_max_push_commits(config.monorepo.max_push_commits) + .with_native_publication(config.mst2.publication_enabled), artifact_service: ArtifactService::new(base.clone(), mock_object_storage()), buck_service: BuckService::mock(), config_handle: ConfigHandle::from_arc(config.clone()), From 8d1267f65d19d33590d94897d054d4590279d137 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 08:33:11 +0800 Subject: [PATCH 05/14] fix(mst2): expose TreeFrame response identity headers --- src/api/router/snapshot_content.rs | 14 ++----- src/api/router/snapshot_router.rs | 59 +++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index b52dbba6..8540f65b 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -14,7 +14,7 @@ use futures::stream::StreamExt; use serde::Deserialize; use serde_json::json; -use super::{abs_view_path, internal, mst2_error_response}; +use super::{abs_view_path, internal, mst2_error_response, treeframe_response}; use crate::ceres::snapshot::{ chunks::{ChunkProjection, get_or_project}, error::{SnapshotError, SnapshotErrorCode}, @@ -290,11 +290,7 @@ pub(super) async fn objects( ); out.extend_from_slice(&end); - axum::response::Response::builder() - .header("content-type", "application/octet-stream") - .header("cache-control", "private, no-cache, no-transform") - .body(axum::body::Body::from(Bytes::from(out))) - .map_err(|e| mst2_error_response(internal(format!("body build failed: {e}")))) + treeframe_response(&snapshot_id, &body, out).map_err(mst2_error_response) } #[derive(Deserialize, Debug)] @@ -589,11 +585,7 @@ pub(super) async fn chunks( ); out.extend_from_slice(&end); - axum::response::Response::builder() - .header("content-type", "application/octet-stream") - .header("cache-control", "private, no-cache, no-transform") - .body(axum::body::Body::from(Bytes::from(out))) - .map_err(|e| mst2_error_response(internal(format!("body build failed: {e}")))) + treeframe_response(&snapshot_id, &body, out).map_err(mst2_error_response) } /// Strict decimal-string parse for unsigned counts (spec 04 §1: no leading diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index 556897e3..4cae9d94 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -18,6 +18,7 @@ use bytes::Bytes; use mst2_codec::descriptor; use serde::Deserialize; use serde_json::json; +use sha2::{Digest, Sha256}; use crate::{ api::MonoApiServiceState, @@ -75,6 +76,33 @@ mod content; /// Spec 14 §4: JSON request bytes hard limit. pub(crate) const JSON_REQUEST_LIMIT: usize = 131_072; +/// The media type is part of the MST/2 TreeFrame wire contract (spec 06 +/// §1). Keep it in one place so every frame-producing endpoint has the same +/// response representation. +pub(crate) const TREEFRAME_MEDIA_TYPE: &str = "application/vnd.mega.treeframe;version=2"; + +/// Build a TreeFrame response with the protocol identity headers. The request +/// digest covers the exact bytes that were parsed, including JSON whitespace +/// and key ordering, so callers must pass the original body. +pub(crate) fn treeframe_response( + snapshot_id: &str, + request_body: &[u8], + body: Vec, +) -> Result { + let request_digest: [u8; 32] = Sha256::digest(request_body).into(); + Response::builder() + .header("content-type", TREEFRAME_MEDIA_TYPE) + .header("x-mega-snapshot-id", snapshot_id) + .header( + "x-mega-request-digest", + format!("sha256:{}", hex_of(&request_digest)), + ) + .header("cache-control", "private, no-cache, no-transform") + .header("vary", "Authorization, Accept") + .body(axum::body::Body::from(Bytes::from(body))) + .map_err(|e| internal(format!("TreeFrame response build failed: {e}"))) +} + tokio::task_local! { /// Per-request id for the error envelope (spec 14 §5). Sourced from the /// global `TraceContext` so the envelope, the `X-Request-Id` response @@ -1136,9 +1164,30 @@ async fn metadata_pages( ); out.extend_from_slice(&end); - Response::builder() - .header("content-type", "application/octet-stream") - .header("cache-control", "private, no-cache, no-transform") - .body(axum::body::Body::from(Bytes::from(out))) - .map_err(|e| mst2_error_response(internal(format!("body build failed: {e}")))) + treeframe_response(&snapshot_id, &body, out).map_err(mst2_error_response) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn treeframe_response_emits_protocol_identity_headers() { + let snapshot_id = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let request_body = br#"{"items":[], "encoding":"identity"}"#; + let response = treeframe_response(snapshot_id, request_body, vec![1, 2, 3]).unwrap(); + + assert_eq!(response.headers()["content-type"], TREEFRAME_MEDIA_TYPE); + assert_eq!(response.headers()["x-mega-snapshot-id"], snapshot_id); + let digest: [u8; 32] = Sha256::digest(request_body).into(); + assert_eq!( + response.headers()["x-mega-request-digest"], + format!("sha256:{}", hex_of(&digest)) + ); + assert_eq!( + response.headers()["cache-control"], + "private, no-cache, no-transform" + ); + assert_eq!(response.headers()["vary"], "Authorization, Accept"); + } } From 829a3bb9bf13d04c8a095df782ec3adb90611d0e Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 09:07:03 +0800 Subject: [PATCH 06/14] ci: run isolated Linux repository gates on pull requests --- .github/workflows/repository-gates.yml | 77 ++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/repository-gates.yml diff --git a/.github/workflows/repository-gates.yml b/.github/workflows/repository-gates.yml new file mode 100644 index 00000000..80a0717e --- /dev/null +++ b/.github/workflows/repository-gates.yml @@ -0,0 +1,77 @@ +name: Repository gates + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: repository-gates-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + linux: + name: Linux repository gates + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + RUST_BACKTRACE: "1" + NO_PROXY: localhost,127.0.0.1,::1,host.docker.internal + no_proxy: localhost,127.0.0.1,::1,host.docker.internal + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + + - uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + + - name: Check formatting + run: cargo +nightly fmt --all --check + + - name: Check all targets and features + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Build + run: cargo build + + - name: Build tests + run: cargo build --tests + + - name: Start isolated test dependencies + shell: bash + run: | + set -euo pipefail + source scripts/lib/mega2-it.sh + mega2_it_ensure_env_test + mega2_it_prepare_git_workdir + docker compose -p mega2-it -f docker/docker-compose.test.yml --profile git \ + up -d --build --wait postgres redis rustfs rustfs-init git-cli + + - name: Run repository tests with test environment + shell: bash + run: | + set -euo pipefail + source .env.test + cargo test --all + + - name: Stop isolated test dependencies + if: always() + run: docker compose -p mega2-it -f docker/docker-compose.test.yml --profile git down -v From 0b03db1d02c31f5ea4c435868f32d43776123b15 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 09:09:26 +0800 Subject: [PATCH 07/14] ci: avoid duplicate case-insensitive workflow environment keys --- .github/workflows/repository-gates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repository-gates.yml b/.github/workflows/repository-gates.yml index 80a0717e..48e7382b 100644 --- a/.github/workflows/repository-gates.yml +++ b/.github/workflows/repository-gates.yml @@ -24,7 +24,6 @@ jobs: CARGO_PROFILE_TEST_DEBUG: "0" RUST_BACKTRACE: "1" NO_PROXY: localhost,127.0.0.1,::1,host.docker.internal - no_proxy: localhost,127.0.0.1,::1,host.docker.internal steps: - uses: actions/checkout@v4 @@ -70,6 +69,7 @@ jobs: run: | set -euo pipefail source .env.test + export no_proxy="$NO_PROXY" cargo test --all - name: Stop isolated test dependencies From 2bdeed8c660dbf949e9d0336c3f9d577812c7a62 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 09:17:39 +0800 Subject: [PATCH 08/14] ci: handle generated future lint and retain failed gate cache --- .github/workflows/repository-gates.yml | 2 ++ src/ceres/pack/import_repo.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/repository-gates.yml b/.github/workflows/repository-gates.yml index 48e7382b..b656c665 100644 --- a/.github/workflows/repository-gates.yml +++ b/.github/workflows/repository-gates.yml @@ -41,6 +41,8 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 + with: + save-if: "true" - name: Check formatting run: cargo +nightly fmt --all --check diff --git a/src/ceres/pack/import_repo.rs b/src/ceres/pack/import_repo.rs index 6b6bd6ce..b98cbb39 100644 --- a/src/ceres/pack/import_repo.rs +++ b/src/ceres/pack/import_repo.rs @@ -452,6 +452,10 @@ impl RepoHandler for ImportRepo { } } +#[allow( + clippy::double_must_use, + reason = "async_recursion adds must_use to the generated boxed Future" +)] #[async_recursion] pub(crate) async fn collect_git_blob_filepaths( storage: GitDbStorage, From c0ef9a118b7cc8e77295cafd3edb0de5c42c95fb Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 09:40:40 +0800 Subject: [PATCH 09/14] fix(mst2): repair native publication compile gates --- src/api/router/snapshot_router.rs | 105 +++++++++--------- .../service/native_publication_push_tests.rs | 3 +- src/jupiter/service/push_queue_service.rs | 3 +- src/jupiter/storage/push_queue_storage.rs | 11 +- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index 9396caa4..e02d72ec 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -365,55 +365,55 @@ async fn resolve(state: State, body: Bytes) -> Result, body: Bytes) -> Result (tempfile::TempDir, PushQueueStorage) { From 650b352e5f37a717898504dad487b9236e99b657 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 10:21:10 +0800 Subject: [PATCH 10/14] test(mst2): include native publication in migration ordering --- src/jupiter/migration/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 3f2c2332..39078944 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -479,13 +479,15 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 3..], + &names[names.len() - 5..], &[ "m20260923_000100_import_repo_cleanups".to_string(), "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), + "m20261005_000100_add_mst2_publication_request_digest".to_string(), + "m20261005_000200_add_mst2_native_head".to_string(), ], - "media paging registered last, after the FU-14/16 migrations" + "native publication registered after media paging and request receipts" ); let db = alias_db().await; From f2ddf8c078f388cd0d0735a10d0928229a944d31 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 10:57:06 +0800 Subject: [PATCH 11/14] test(mst2): pin receipt migration fixture by identity --- src/jupiter/storage/mst2_publication_tests.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/jupiter/storage/mst2_publication_tests.rs b/src/jupiter/storage/mst2_publication_tests.rs index bb743a44..5aec659e 100644 --- a/src/jupiter/storage/mst2_publication_tests.rs +++ b/src/jupiter/storage/mst2_publication_tests.rs @@ -762,18 +762,26 @@ async fn noop_reservation_cannot_move_to_another_transaction() { async fn migration_preserves_legacy_receipts_and_validates_digest_columns() { let temp = tempfile::tempdir().unwrap(); let db = test_db_connection(temp.path()).await; - let old_count = u32::try_from(Migrator::migrations().len() - 1).unwrap(); + let migrations = Migrator::migrations(); + let receipt_at = migrations + .iter() + .position(|migration| { + migration.name() == "m20261005_000100_add_mst2_publication_request_digest" + }) + .expect("publication receipt migration registered"); + let old_count = u32::try_from(receipt_at).unwrap(); Migrator::up(&db, Some(old_count)).await.unwrap(); db.execute_unprepared( "INSERT INTO mst2_publication (operation_id, namespace, sequence, old_oid, new_oid, writer_epoch, writer_kind, created_at) \ VALUES ('legacy', '/', 1, 'a', 'b', 1, 'test', now())" ).await.unwrap(); apply_migrations(&db, false).await.unwrap(); - let latest = Migrator::migrations().pop().unwrap(); - latest - .up(&sea_orm_migration::SchemaManager::new(&db)) + let txn = db.begin().await.unwrap(); + migrations[receipt_at] + .up(&sea_orm_migration::SchemaManager::new(&txn)) .await .unwrap(); + txn.commit().await.unwrap(); let legacy = mst2_publication::Entity::find() .one(&db) .await From da847dc4cd9484b926c5c71faf190d521c2c948d Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 10:57:28 +0800 Subject: [PATCH 12/14] test(mst2): persist real blobs for native HTTP projection --- .../service/native_publication_push_tests.rs | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/jupiter/service/native_publication_push_tests.rs b/src/jupiter/service/native_publication_push_tests.rs index 6ebb703c..f24f591c 100644 --- a/src/jupiter/service/native_publication_push_tests.rs +++ b/src/jupiter/service/native_publication_push_tests.rs @@ -18,18 +18,29 @@ async fn native_fixture() -> (tempfile::TempDir, crate::jupiter::storage::Storag let storage = crate::jupiter::tests::test_storage_with_config(temp.path(), config).await; let storage = crate::jupiter::tests::with_test_vault(storage, temp.path()).await; let name = format!("native-{}", uuid::Uuid::new_v4().simple()); - let (tip, path) = wh03_path_fixture(&storage, &name).await; - // Persist the literal file bytes expected by the tree fixture for actual HTTP projection. - for (oid, raw) in [("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", b"keep".as_slice()), - ("dddddddddddddddddddddddddddddddddddddddd", b"old".as_slice())] { - storage.git_service.save_object_from_raw(Bytes::copy_from_slice(raw)).await.unwrap(); - storage.mono_storage().get_connection().execute_raw(Statement::from_sql_and_values( - storage.mono_storage().get_connection().get_database_backend(), - "INSERT INTO mst2_verified_object(storage_domain,git_oid,object_kind,raw_sha256,size,verification_version,state,created_at) \ - VALUES ('git',$1,'blob',$2,$3,2,'VERIFIED',now()) ON CONFLICT DO NOTHING", - [oid.into(), Sha256::digest(raw).to_vec().into(), (raw.len() as i64).into()], - )).await.unwrap(); - } + use git_internal::internal::object::{ + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }; + let keep_oid = storage.git_service.save_object_from_raw(Bytes::from_static(b"keep")).await.unwrap(); + let old_oid = storage.git_service.save_object_from_raw(Bytes::from_static(b"old")).await.unwrap(); + let child = Tree::from_tree_items(vec![wh03_blob_item("x.txt", &old_oid)]).unwrap(); + let root_tree = Tree::from_tree_items(vec![ + wh03_blob_item(".gitkeep", &keep_oid), + TreeItem::new(TreeItemMode::Tree, child.id, name.clone()), + ]).unwrap(); + let root_commit = Commit::from_tree_id(root_tree.id, vec![], "root"); + let tip = Commit::from_tree_id(child.id, vec![], "path tip"); + let mono = storage.mono_storage(); + mono.save_mega_trees(vec![child.clone(), root_tree.clone()], root_commit.id, None).await.unwrap(); + mono.save_mega_commits(vec![root_commit.clone(), tip.clone()], None).await.unwrap(); + mono.save_refs(mega_refs::Model::new( + "/", MEGA_BRANCH_NAME.to_owned(), root_commit.id.to_string(), root_tree.id.to_string(), false, + ), None).await.unwrap(); + let path = format!("/{name}"); + mono.save_refs(mega_refs::Model::new( + path.clone(), MEGA_BRANCH_NAME.to_owned(), tip.id.to_string(), child.id.to_string(), false, + ), None).await.unwrap(); storage.mono_storage().initialize_native_publication(NATIVE_INSTANCE).await.unwrap(); (temp, storage, tip, path) } @@ -219,8 +230,9 @@ async fn http_resolve_captured_before_commit_never_mixes_new_sequence_with_old_d assert!(matches!(wh03_exec(&storage,id).await,ExecuteOutcome::Done{..})); let state=api_state(storage.clone()).await; let (status,v1)=http_resolve(state.clone(),"/").await; - assert_eq!(status,200); - let (_,subscope)=http_resolve(state.clone(),&path).await; + assert_eq!(status,200,"{v1}"); + let (status,subscope)=http_resolve(state.clone(),&path).await; + assert_eq!(status,200,"{subscope}"); assert_eq!(v1["publication_sequence"],subscope["publication_sequence"]); assert_eq!(v1["writer_epoch"],subscope["writer_epoch"]); let root_v1=storage.mono_storage().get_main_ref("/").await.unwrap().unwrap(); @@ -234,14 +246,15 @@ async fn http_resolve_captured_before_commit_never_mixes_new_sequence_with_old_d tokio::time::timeout(Duration::from_secs(5),captured.wait()).await.unwrap(); let first_commit=storage.mono_storage().get_commit_by_hash(&first).await.unwrap().unwrap(); let first_commit=git_internal::internal::object::commit::Commit::from_mega_model(first_commit); - let (next,next_payload)=wh03_save_n1_commit(&storage,first_commit.id,"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","next projection").await; + let next_blob = storage.git_service.save_object_from_raw(Bytes::from_static(b"next projection")).await.unwrap(); + let (next,next_payload)=wh03_save_n1_commit(&storage,first_commit.id,&next_blob,"next projection").await; let id=wh03_enqueue_push(&storage,&path,&first,&next,&next_payload).await; assert!(matches!(wh03_exec(&storage,id).await,ExecuteOutcome::Done{..})); tokio::time::timeout(Duration::from_secs(5),release.wait()).await.unwrap(); let (status,delayed)=tokio::time::timeout(Duration::from_secs(10),&mut reader).await.unwrap().unwrap(); - assert_eq!(status,200);assert_eq!(delayed["descriptor"],v1["descriptor"]); + assert_eq!(status,200,"{delayed}");assert_eq!(delayed["descriptor"],v1["descriptor"]); assert_eq!(delayed["publication_sequence"],v1["publication_sequence"]); let (status,v2)=http_resolve(state,"/").await; - assert_eq!(status,200);assert_eq!(v2["publication_sequence"],"2"); + assert_eq!(status,200,"{v2}");assert_eq!(v2["publication_sequence"],"2"); assert_ne!(v2["descriptor"]["snapshot_id"],v1["descriptor"]["snapshot_id"]); } From 6b0b33b62ed590bb910da6207a1619c7a76e9a5d Mon Sep 17 00:00:00 2001 From: Xiaoyang Han Date: Mon, 5 Oct 2026 18:13:25 +0800 Subject: [PATCH 13/14] test(mst2): gate the integrated native commit update candidate (#38) * fix(mst2): make retention acquisition atomic Protect snapshot retention roots and edges with one validated commit, reject DELETING references, and propagate lease lifecycle failures. * feat(mst2): add durable retention GC schema Add SeaORM retention entities and a forward migration for incoming reference counts and idempotent GC operation replay. * test(mst2): include retention migration in ordering check * fix(mst2): bound JSON body reads and preserve typed rejections * fix(mst2): reject duplicate JSON keys including null and escapes * feat(mst2): add atomic PostgreSQL retention repository * fix(mst2): remove obsolete body extractor import * test(mst2): account for integrated native and retention migrations * fix(mst2): repair native publication compile gates * test(mst2): require successful bounded-input lease fixture registration * test(mst2): seed retention migration timestamps * test(mst2): seed legacy retention edge timestamps * test(mst2): assert the GC receipt completion constraint --- src/api/router/snapshot_content.rs | 7 +- src/api/router/snapshot_request.rs | 114 ++++ src/api/router/snapshot_request_tests.rs | 353 ++++++++++ src/api/router/snapshot_router.rs | 78 ++- src/callisto/mod.rs | 4 + src/callisto/mst2_retention_edge.rs | 19 + src/callisto/mst2_retention_gc_op.rs | 26 + src/callisto/mst2_retention_node.rs | 25 + src/callisto/mst2_retention_root.rs | 20 + src/callisto/prelude.rs | 4 + src/ceres/snapshot/error.rs | 5 +- ...05_000100_add_mst2_retention_durability.rs | 119 ++++ ...1005_000200_harden_mst2_retention_graph.rs | 76 +++ src/jupiter/migration/mod.rs | 10 +- src/jupiter/storage/mod.rs | 1 + src/jupiter/storage/mst2_retention.rs | 633 ++++++++++++++++++ src/jupiter/storage/mst2_retention_tests.rs | 629 +++++++++++++++++ 17 files changed, 2089 insertions(+), 34 deletions(-) create mode 100644 src/api/router/snapshot_request.rs create mode 100644 src/api/router/snapshot_request_tests.rs create mode 100644 src/callisto/mst2_retention_edge.rs create mode 100644 src/callisto/mst2_retention_gc_op.rs create mode 100644 src/callisto/mst2_retention_node.rs create mode 100644 src/callisto/mst2_retention_root.rs create mode 100644 src/jupiter/migration/m20261005_000100_add_mst2_retention_durability.rs create mode 100644 src/jupiter/migration/m20261005_000200_harden_mst2_retention_graph.rs create mode 100644 src/jupiter/storage/mst2_retention.rs create mode 100644 src/jupiter/storage/mst2_retention_tests.rs diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index 8540f65b..f06fd35b 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -9,12 +9,11 @@ use axum::{ http::HeaderMap, response::{IntoResponse, Response}, }; -use bytes::Bytes; use futures::stream::StreamExt; use serde::Deserialize; use serde_json::json; -use super::{abs_view_path, internal, mst2_error_response, treeframe_response}; +use super::{abs_view_path, internal, mst2_error_response, request::Mst2Bytes, treeframe_response}; use crate::ceres::snapshot::{ chunks::{ChunkProjection, get_or_project}, error::{SnapshotError, SnapshotErrorCode}, @@ -174,7 +173,7 @@ const OBJECT_TOTAL_MAX: usize = 8 * 1024 * 1024; pub(super) async fn objects( state: State, AxumPath(snapshot_id): AxumPath, - body: Bytes, + Mst2Bytes(body): Mst2Bytes, ) -> Result { ensure(&state)?; let ctx = runtime() @@ -469,7 +468,7 @@ struct Planned { pub(super) async fn chunks( state: State, AxumPath(snapshot_id): AxumPath, - body: Bytes, + Mst2Bytes(body): Mst2Bytes, ) -> Result { ensure(&state)?; let ctx = runtime() diff --git a/src/api/router/snapshot_request.rs b/src/api/router/snapshot_request.rs new file mode 100644 index 00000000..ec43af70 --- /dev/null +++ b/src/api/router/snapshot_request.rs @@ -0,0 +1,114 @@ +//! Bounded raw JSON input for the MST/2 POST surface (spec 14). + +use std::{collections::HashSet, fmt, time::Duration}; + +use axum::{ + extract::{FromRequest, Request}, + http::StatusCode, +}; +use bytes::Bytes; +use serde::{ + Deserialize, Deserializer, + de::{self, MapAccess, SeqAccess, Visitor}, +}; + +use crate::ceres::snapshot::error::{SnapshotError, SnapshotErrorCode}; + +/// One overall read deadline, including a body which keeps trickling bytes. +/// This bounds input collection only, not handler work or response streams. +pub(super) const JSON_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); + +/// Preserve the original bytes for TreeFrame request-body digests. The router +/// supplies DefaultBodyLimit; its rejection is converted to the MST envelope. +pub(super) struct Mst2Bytes(pub(super) Bytes); + +/// Decode keys before comparing them, including escaped spellings. This +/// separate pass also catches duplicate optional fields whose first value is +/// null, which a derived DTO can otherwise treat as an absent field. +pub(super) fn validate_json_keys(body: &[u8]) -> Result<(), serde_json::Error> { + serde_json::from_slice::(body).map(|_| ()) +} + +struct UniqueKeys; + +impl<'de> Deserialize<'de> for UniqueKeys { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(UniqueKeyVisitor) + } +} + +struct UniqueKeyVisitor; + +impl<'de> Visitor<'de> for UniqueKeyVisitor { + type Value = UniqueKeys; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object keys") + } + + fn visit_bool(self, _: bool) -> Result { + Ok(UniqueKeys) + } + fn visit_i64(self, _: i64) -> Result { + Ok(UniqueKeys) + } + fn visit_u64(self, _: u64) -> Result { + Ok(UniqueKeys) + } + fn visit_f64(self, _: f64) -> Result { + Ok(UniqueKeys) + } + fn visit_str(self, _: &str) -> Result { + Ok(UniqueKeys) + } + fn visit_unit(self) -> Result { + Ok(UniqueKeys) + } + + fn visit_seq>(self, mut sequence: A) -> Result { + while sequence.next_element::()?.is_some() {} + Ok(UniqueKeys) + } + + fn visit_map>(self, mut object: A) -> Result { + let mut keys = HashSet::new(); + while let Some(key) = object.next_key::()? { + if !keys.insert(key) { + return Err(de::Error::custom("duplicate JSON object key")); + } + object.next_value::()?; + } + Ok(UniqueKeys) + } +} + +impl FromRequest for Mst2Bytes +where + S: Send + Sync, +{ + type Rejection = SnapshotError; + + async fn from_request(req: Request, state: &S) -> Result { + match tokio::time::timeout(JSON_REQUEST_TIMEOUT, Bytes::from_request(req, state)).await { + Ok(Ok(body)) => Ok(Self(body)), + Ok(Err(error)) => { + let (code, message) = if error.status() == StatusCode::PAYLOAD_TOO_LARGE { + ( + SnapshotErrorCode::LimitExceeded, + "request body over the spec 14 limit", + ) + } else { + ( + SnapshotErrorCode::InvalidRequest, + "could not read request body", + ) + }; + Err(SnapshotError::new(code, message)) + } + Err(_) => Err(SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "request body read deadline exceeded", + )), + } + } +} diff --git a/src/api/router/snapshot_request_tests.rs b/src/api/router/snapshot_request_tests.rs new file mode 100644 index 00000000..fc577c85 --- /dev/null +++ b/src/api/router/snapshot_request_tests.rs @@ -0,0 +1,353 @@ +use std::{ + convert::Infallible, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + task::{Context, Poll}, + time::Duration, +}; + +use axum::{ + Router, + body::{Body, to_bytes}, + http::Request, + middleware, + response::Response, + routing::get, +}; +use bytes::Bytes; +use futures::{Stream, stream}; +use serde_json::Value; +use tower::ServiceExt; + +use super::{JSON_REQUEST_LIMIT, request::JSON_REQUEST_TIMEOUT, routers}; +use crate::{ + api::{MonoApiServiceState, oauth::api_store::BrowserSessionStore}, + ceres::{ + api_service::cache::GitObjectCache, + snapshot::{descriptor::build, runtime::runtime, view::SnapshotView}, + }, + config::testing::isolated_config, + jupiter::tests::{test_redis_manager, test_storage_with_config}, + server::trace_context::{TraceContext, inject_trace_context}, +}; + +struct Fixture { + app: Router, + snapshot_id: String, + lease_id: String, + expires: u64, + _temp: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> Self { + let temp = tempfile::TempDir::new().unwrap(); + let mut config = isolated_config(temp.path().join("config")); + config.mst2.enabled = true; + config.mst2.instance_uuid = Some(uuid::Uuid::new_v4().to_string()); + config.mst2.auth_token = Some("mst2-input-test".to_string()); + let storage = test_storage_with_config(temp.path(), config).await; + let state = MonoApiServiceState { + entity_store: storage.entity_store.clone(), + storage, + session_store: BrowserSessionStore::Anonymous, + git_object_cache: Arc::new(GitObjectCache { + connection: test_redis_manager().await, + prefix: String::new(), + }), + listen_addr: "127.0.0.1:0".to_string(), + }; + let view = SnapshotView::from_commit(&"1".repeat(40), &"2".repeat(40)); + let built = build(&state.storage.config().mst2, &view, "/", [3; 32]).unwrap(); + let ctx = runtime() + .insert_context(built, &view.commit_oid, &view.root_tree_oid, 60) + .expect("fixture lease registration must succeed"); + // Reproduce the server's nest-after-layer order: MST must establish + // its own request context even when the earlier layer does not run. + let app = Router::new() + .route("/outside", get(|| async { "ok" })) + .layer(middleware::from_fn(inject_trace_context)) + .nest("/api/v2", routers(state.clone()).with_state(state)); + Self { + app, + snapshot_id: ctx.built.snapshot_id, + lease_id: ctx.lease_id, + expires: ctx.lease_expires_at_unix, + _temp: temp, + } + } + + fn request(&self, suffix: &str, body: Body) -> Request { + Request::builder() + .method("POST") + .uri(format!("/api/v2/snapshots/{suffix}")) + .header("authorization", "Bearer mst2-input-test") + .header("x-mega-snapshot-lease", &self.lease_id) + .header("content-type", "application/json") + .body(body) + .unwrap() + } + + fn renew_path(&self) -> String { + format!("leases/{}/renew", self.lease_id) + } + + fn assert_lease_unchanged(&self) { + let ctx = runtime().context(&self.snapshot_id).unwrap(); + assert_eq!(ctx.lease_id, self.lease_id); + assert_eq!(ctx.lease_expires_at_unix, self.expires); + } +} + +async fn assert_error(response: Response, status: u16, code: &str, retryable: bool) -> Value { + assert_eq!(response.status().as_u16(), status); + assert_eq!(response.headers()["content-type"], "application/json"); + let request_id = response.headers()["x-request-id"] + .to_str() + .unwrap() + .to_string(); + assert!(!request_id.is_empty()); + let body = to_bytes(response.into_body(), 16 * 1024).await.unwrap(); + let value: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["error"]["code"], code); + assert_eq!(value["error"]["request_id"], request_id); + assert_eq!(value["error"]["retryable"], retryable); + assert!(!value["error"]["message"].as_str().unwrap().is_empty()); + value +} + +fn chunked(bytes: Vec) -> Body { + let chunks: Vec> = bytes + .chunks(4096) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + Body::from_stream(stream::iter(chunks)) +} + +#[tokio::test] +async fn mst2_json_actual_chunked_limit_has_typed_errors_on_every_post() { + let fixture = Fixture::new().await; + let paths = [ + "resolve".to_string(), + fixture.renew_path(), + format!("{}/lookup", fixture.snapshot_id), + format!("{}/metadata/pages", fixture.snapshot_id), + format!("{}/objects", fixture.snapshot_id), + format!("{}/chunks", fixture.snapshot_id), + ]; + for path in paths { + for declared in [None, Some("2")] { + let mut request = fixture.request(&path, chunked(vec![b' '; JSON_REQUEST_LIMIT + 1])); + if let Some(length) = declared { + request + .headers_mut() + .insert("content-length", length.parse().unwrap()); + } + let response = fixture.app.clone().oneshot(request).await.unwrap(); + assert_error(response, 413, "LIMIT_EXCEEDED", false).await; + fixture.assert_lease_unchanged(); + } + } + + let mut body = br#"{"lease_seconds":120}"#.to_vec(); + body.resize(JSON_REQUEST_LIMIT, b' '); + let response = fixture + .app + .clone() + .oneshot(fixture.request(&fixture.renew_path(), chunked(body))) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert!(!response.headers()["x-request-id"].is_empty()); + let renewed = runtime().context(&fixture.snapshot_id).unwrap(); + assert!(renewed.lease_expires_at_unix > fixture.expires); +} + +#[tokio::test] +async fn mst2_json_declared_oversize_is_rejected_without_reading_body() { + let fixture = Fixture::new().await; + let polls = Arc::new(AtomicUsize::new(0)); + let observed = polls.clone(); + let body = Body::from_stream(stream::poll_fn(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + Poll::>>::Pending + })); + let mut request = fixture.request(&fixture.renew_path(), body); + request.headers_mut().insert( + "content-length", + (JSON_REQUEST_LIMIT + 1).to_string().parse().unwrap(), + ); + let response = fixture.app.clone().oneshot(request).await.unwrap(); + assert_error(response, 413, "LIMIT_EXCEEDED", false).await; + assert_eq!(polls.load(Ordering::SeqCst), 0); + fixture.assert_lease_unchanged(); +} + +#[tokio::test] +async fn mst2_json_closed_dtos_duplicates_and_malformed_input_do_not_renew() { + let fixture = Fixture::new().await; + let renew = fixture.renew_path(); + let lookup = format!("{}/lookup", fixture.snapshot_id); + let metadata = format!("{}/metadata/pages", fixture.snapshot_id); + let objects = format!("{}/objects", fixture.snapshot_id); + let chunks = format!("{}/chunks", fixture.snapshot_id); + let cases: Vec<(&str, &[u8])> = vec![ + ("resolve", br#"{"target":{"kind":"latest"},"scope":"/","scope":"/"}"#), + ("resolve", br#"{"target":{"kind":"latest","kind":"latest"}}"#), + ("resolve", br#"{"target":{"kind":"latest","extra":true}}"#), + ("resolve", br#"{"target":{"kind":"latest"},"extra":true}"#), + (&renew, br#"{"lease_seconds":3600,"lease_seconds":120}"#), + (&renew, br#"{"lease_seconds":3600,"\u006cease_seconds":120}"#), + (&renew, br#"{"lease_seconds":null,"lease_seconds":120}"#), + (&renew, br#"{"lease_seconds":null,"\u006cease_seconds":120}"#), + (&renew, br#"{"lease_seconds":null,"lease_seconds":null}"#), + (&renew, br#"{"lease_seconds":3600,"extra":true}"#), + (&renew, br#"{"lease_seconds":"120"}"#), + (&renew, br#"{"lease_seconds":1.5}"#), + (&renew, br#"{"lease_seconds":-1}"#), + (&renew, br#"{"lease_seconds":NaN}"#), + (&renew, br#"{"lease_seconds":Infinity}"#), + (&renew, br#"{"lease_seconds":18446744073709551616}"#), + (&renew, br#"{"lease_seconds":"#), + (&renew, b"{\"lease_seconds\":\xff}"), + (&renew, b"{\"lease_seconds\":3600} {}"), + (&renew, b"{\"lease_seconds\":3600} x"), + (&lookup, br#"{"paths":[],"paths":[]}"#), + (&lookup, br#"{"paths":[],"extra":true}"#), + (&metadata, br#"{"items":[],"items":[]}"#), + (&metadata, br#"{"items":[],"extra":true}"#), + (&metadata, br#"{"items":[{"directory_path":"/","route":[],"route":[]}]}"#), + (&metadata, br#"{"items":[{"directory_path":"/","extra":true}]}"#), + (&objects, br#"{"items":[],"items":[]}"#), + (&objects, br#"{"items":[],"extra":true}"#), + (&objects, br#"{"items":[{"path":"/a","path":"/b","expected_digest":"x"}]}"#), + (&objects, br#"{"items":[{"path":"/a","expected_digest":"x","extra":true}]}"#), + (&chunks, br#"{"items":[],"items":[]}"#), + (&chunks, br#"{"items":[],"extra":true}"#), + (&chunks, br#"{"items":[{"path":"/a","expected_digest":"x","map_id":"x","chunk_index":"0","chunk_index":"1"}]}"#), + (&chunks, br#"{"items":[{"path":"/a","expected_digest":"x","map_id":"x","chunk_index":"0","extra":true}]}"#), + ]; + for (path, body) in cases { + let response = fixture + .app + .clone() + .oneshot(fixture.request(path, Body::from(body.to_vec()))) + .await + .unwrap(); + assert_error(response, 400, "INVALID_REQUEST", false).await; + fixture.assert_lease_unchanged(); + } + + for body in [Body::empty(), Body::from("{} \r\n\t")] { + let response = fixture + .app + .clone() + .oneshot(fixture.request(&renew, body)) + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert!(!response.headers()["x-request-id"].is_empty()); + } +} + +struct SlowBody { + interval: Option, + chunks: Arc, + dropped: Arc, +} + +impl Stream for SlowBody { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + match &mut this.interval { + Some(interval) => match interval.poll_tick(cx) { + Poll::Ready(_) => { + this.chunks.fetch_add(1, Ordering::SeqCst); + Poll::Ready(Some(Ok(Bytes::from_static(b" ")))) + } + Poll::Pending => Poll::Pending, + }, + None => Poll::Pending, + } + } +} + +impl Drop for SlowBody { + fn drop(&mut self) { + self.dropped.store(true, Ordering::SeqCst); + } +} + +#[tokio::test] +async fn mst2_json_real_post_pending_and_trickle_bodies_hit_overall_deadline() { + let fixture = Fixture::new().await; + let exercise = |trickle: bool| { + let app = fixture.app.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let chunks = Arc::new(AtomicUsize::new(0)); + let body = Body::from_stream(SlowBody { + interval: trickle.then(|| tokio::time::interval(Duration::from_millis(50))), + chunks: chunks.clone(), + dropped: dropped.clone(), + }); + let request = fixture.request(&fixture.renew_path(), body); + async move { + let response = tokio::time::timeout( + JSON_REQUEST_TIMEOUT + Duration::from_secs(5), + app.oneshot(request), + ) + .await + .expect("MST input deadline must terminate the request") + .unwrap(); + assert_error(response, 503, "TEMPORARY_UNAVAILABLE", true).await; + assert!(dropped.load(Ordering::SeqCst)); + if trickle { + assert!(chunks.load(Ordering::SeqCst) > 1); + } else { + assert_eq!(chunks.load(Ordering::SeqCst), 0); + } + } + }; + tokio::join!(exercise(false), exercise(true)); + fixture.assert_lease_unchanged(); + let response = fixture + .app + .clone() + .oneshot(fixture.request(&fixture.renew_path(), Body::from("{}"))) + .await + .unwrap(); + assert_eq!(response.status(), 200); +} + +#[tokio::test] +async fn mst2_json_body_read_failure_is_typed_and_preserves_request_context() { + let fixture = Fixture::new().await; + let body = Body::from_stream(stream::once(async { + Err::(std::io::Error::other("private backend key must not leak")) + })); + let mut request = fixture.request(&fixture.renew_path(), body); + request + .headers_mut() + .insert("x-request-id", "different-inbound-id".parse().unwrap()); + request.extensions_mut().insert(TraceContext { + trace_id: Arc::from("existing-trace-id"), + }); + let response = fixture.app.clone().oneshot(request).await.unwrap(); + let value = assert_error(response, 400, "INVALID_REQUEST", false).await; + assert_eq!(value["error"]["request_id"], "existing-trace-id"); + assert_eq!(value["error"]["message"], "could not read request body"); + fixture.assert_lease_unchanged(); + + let mut request = fixture.request(&fixture.renew_path(), Body::from("{")); + request + .headers_mut() + .insert("x-request-id", "accepted-inbound-id".parse().unwrap()); + let response = fixture.app.clone().oneshot(request).await.unwrap(); + let value = assert_error(response, 400, "INVALID_REQUEST", false).await; + assert_eq!(value["error"]["request_id"], "accepted-inbound-id"); +} diff --git a/src/api/router/snapshot_router.rs b/src/api/router/snapshot_router.rs index 7e8f9e0e..9330837b 100644 --- a/src/api/router/snapshot_router.rs +++ b/src/api/router/snapshot_router.rs @@ -16,6 +16,7 @@ use axum::{ use base64::Engine; use bytes::Bytes; use mst2_codec::descriptor; +use request::Mst2Bytes; use serde::Deserialize; use serde_json::json; use sha2::{Digest, Sha256}; @@ -73,6 +74,13 @@ pub fn routers(api_state: MonoApiServiceState) -> Router { #[path = "snapshot_content.rs"] mod content; +#[path = "snapshot_request.rs"] +mod request; + +#[cfg(test)] +#[path = "snapshot_request_tests.rs"] +mod request_tests; + /// Spec 14 §4: JSON request bytes hard limit. pub(crate) const JSON_REQUEST_LIMIT: usize = 131_072; @@ -141,28 +149,36 @@ const LEASE_HEADER: &str = "x-mega-snapshot-lease"; /// knowing the snapshot id alone is not a capability. async fn snapshot_auth_middleware( State(state): State, - req: axum::extract::Request, + mut req: axum::extract::Request, next: axum::middleware::Next, ) -> Response { - // Same id the global trace layer echoes on responses and logs. + // Nested routes may be added after the server's trace layer. Reuse an + // existing context or establish one here, including rejection responses. let id = req .extensions() .get::() - .map(|c| c.trace_id.to_string()) - .unwrap_or_default(); - REQUEST_ID - .scope(id, async { + .map(|c| c.trace_id.clone()) + .unwrap_or_else(|| crate::server::trace_context::resolve_trace_id(req.headers())); + req.extensions_mut() + .insert(crate::server::trace_context::TraceContext { + trace_id: id.clone(), + }); + let mut response = REQUEST_ID + .scope(id.to_string(), async { if let Some(res) = auth_error(&state, req.headers(), req.uri().path()) { return res; } next.run(req).await }) - .await + .await; + if let Ok(value) = HeaderValue::from_str(&id) { + response.headers_mut().insert("x-request-id", value); + } + response } -/// Spec 14 §4 enforcement with the MST/2 error envelope (DefaultBodyLimit's -/// own rejection is plain-text). Content-Length is checked here; a lying -/// chunked body still trips DefaultBodyLimit inside the extractor. +/// Reject an oversized declared length before consuming any body. Actual +/// bytes and the overall read deadline are checked by `Mst2Bytes`. async fn reject_oversize_body( req: axum::extract::Request, next: axum::middleware::Next, @@ -186,12 +202,14 @@ async fn reject_oversize_body( /// (spec 14 §5 INVALID_REQUEST). Size is enforced by the router layers. #[allow(clippy::result_large_err)] pub(crate) fn parse_json_body(body: &Bytes) -> Result { - serde_json::from_slice(body).map_err(|e| { - mst2_error_response(SnapshotError::new( - SnapshotErrorCode::InvalidRequest, - format!("malformed request body: {e}"), - )) - }) + request::validate_json_keys(body) + .and_then(|()| serde_json::from_slice(body)) + .map_err(|e| { + mst2_error_response(SnapshotError::new( + SnapshotErrorCode::InvalidRequest, + format!("malformed request body: {e}"), + )) + }) } /// Bearer-token check shared by the auth middleware; the parsing rule is the @@ -260,7 +278,9 @@ fn mst2_error_response(err: SnapshotError) -> Response { "request_id": current_request_id(), "retryable": matches!( err.code, - SnapshotErrorCode::SnapshotNotReady | SnapshotErrorCode::Internal + SnapshotErrorCode::SnapshotNotReady + | SnapshotErrorCode::TemporaryUnavailable + | SnapshotErrorCode::Internal ), } })), @@ -274,6 +294,12 @@ impl From for Response { } } +impl IntoResponse for SnapshotError { + fn into_response(self) -> Response { + mst2_error_response(self) + } +} + /// MegaError → snapshot error: storage/lease failures must surface as errors, /// never as absence (spec 00 SYS-04). fn internal(e: E) -> SnapshotError { @@ -363,7 +389,10 @@ fn ensure_enabled(state: &MonoApiServiceState) -> Result<(), SnapshotError> { // `lfs_router::enforce_lfs_access` (where the error is the rare arm and is // boxed), there is nothing to gain here, so the lint is allowed outright. #[allow(clippy::result_large_err)] -async fn resolve(state: State, body: Bytes) -> Result { +async fn resolve( + state: State, + Mst2Bytes(body): Mst2Bytes, +) -> Result { ensure_enabled(&state).map_err(mst2_error_response)?; let req: ResolveRequest = parse_json_body(&body)?; // Unknown target kinds are client errors, never a silent fallback to @@ -540,18 +569,13 @@ struct RenewRequest { async fn lease_renew( state: State, AxumPath(lease_id): AxumPath, - body: Bytes, + Mst2Bytes(body): Mst2Bytes, ) -> Result { ensure_enabled(&state).map_err(mst2_error_response)?; let req: RenewRequest = if body.is_empty() { RenewRequest::default() } else { - serde_json::from_slice(&body).map_err(|e| { - mst2_error_response(SnapshotError::new( - SnapshotErrorCode::ScopeInvalid, - format!("malformed renew body: {e}"), - )) - })? + parse_json_body(&body)? }; let renewed = runtime() .renew_lease(&lease_id, req.lease_seconds.unwrap_or(600)) @@ -911,7 +935,7 @@ async fn lookup( state: State, AxumPath(snapshot_id): AxumPath, _headers: HeaderMap, - body: Bytes, + Mst2Bytes(body): Mst2Bytes, ) -> Result { ensure_enabled(&state).map_err(mst2_error_response)?; let req: LookupRequest = parse_json_body(&body)?; @@ -1068,7 +1092,7 @@ async fn metadata_pages( state: State, AxumPath(snapshot_id): AxumPath, _headers: HeaderMap, - body: Bytes, + Mst2Bytes(body): Mst2Bytes, ) -> Result { ensure_enabled(&state).map_err(mst2_error_response)?; let req: MetadataPagesRequest = parse_json_body(&body)?; diff --git a/src/callisto/mod.rs b/src/callisto/mod.rs index 8d8e5440..ba9673e6 100644 --- a/src/callisto/mod.rs +++ b/src/callisto/mod.rs @@ -63,6 +63,10 @@ pub mod mst2_native_publication; pub mod mst2_publication; pub mod mst2_publication_outbox; pub mod mst2_queue_noop_receipt; +pub mod mst2_retention_edge; +pub mod mst2_retention_gc_op; +pub mod mst2_retention_node; +pub mod mst2_retention_root; pub mod mst2_verified_object; pub mod notification_event_types; pub mod oci_blob_ref; diff --git a/src/callisto/mst2_retention_edge.rs b/src/callisto/mst2_retention_edge.rs new file mode 100644 index 00000000..2c861bdf --- /dev/null +++ b/src/callisto/mst2_retention_edge.rs @@ -0,0 +1,19 @@ +//! MST/2 retention graph edges (spec 10 §6). + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_retention_edge")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true)] + pub id: i64, + pub parent_id: String, + pub child_id: String, + pub created_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/mst2_retention_gc_op.rs b/src/callisto/mst2_retention_gc_op.rs new file mode 100644 index 00000000..833c949d --- /dev/null +++ b/src/callisto/mst2_retention_gc_op.rs @@ -0,0 +1,26 @@ +//! Durable MST/2 GC operation log (spec 10 §6 crash replay). + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_retention_gc_op")] +pub struct Model { + /// Caller supplied idempotency key. A retry of one physical GC step must + /// address the same row, so replay cannot decrement an edge twice. + #[sea_orm(primary_key, auto_increment = false)] + pub operation_id: String, + pub node_id: String, + /// `MARK_DELETING` or `REMOVE`. + pub operation: String, + /// `PENDING`, `APPLIED`, or `FAILED`; workers may replay `PENDING`. + pub state: String, + pub attempts: i32, + pub created_at: DateTimeWithTimeZone, + pub completed_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/mst2_retention_node.rs b/src/callisto/mst2_retention_node.rs new file mode 100644 index 00000000..40ce2999 --- /dev/null +++ b/src/callisto/mst2_retention_node.rs @@ -0,0 +1,25 @@ +//! MST/2 retention graph nodes (spec 10 §6). + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_retention_node")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub node_id: String, + pub kind: String, + pub state: String, + #[sea_orm(column_type = "BigInteger")] + pub bytes: i64, + /// Number of unique incoming retention edges. Root coverage is kept in + /// `mst2_retention_root` and checked in the same transaction. + #[sea_orm(column_type = "BigInteger")] + pub incoming_refs: i64, + pub created_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/mst2_retention_root.rs b/src/callisto/mst2_retention_root.rs new file mode 100644 index 00000000..9b60cb5a --- /dev/null +++ b/src/callisto/mst2_retention_root.rs @@ -0,0 +1,20 @@ +//! MST/2 retention roots (spec 10 §5/§6). + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "mst2_retention_root")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true)] + pub id: i64, + pub node_id: String, + pub root_key: String, + pub root_kind: String, + pub created_at: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/callisto/prelude.rs b/src/callisto/prelude.rs index 7a4cd099..33e929b9 100644 --- a/src/callisto/prelude.rs +++ b/src/callisto/prelude.rs @@ -20,6 +20,10 @@ pub use super::{ mega_tag::Entity as MegaTag, mega_tree::Entity as MegaTree, mega_webhook::Entity as MegaWebhook, mega_webhook_delivery::Entity as MegaWebhookDelivery, mega_webhook_event_type::Entity as MegaWebhookEventType, + mst2_retention_edge::Entity as Mst2RetentionEdge, + mst2_retention_gc_op::Entity as Mst2RetentionGcOp, + mst2_retention_node::Entity as Mst2RetentionNode, + mst2_retention_root::Entity as Mst2RetentionRoot, notification_event_types::Entity as NotificationEventTypes, oci_blob_ref::Entity as OciBlobRef, oci_manifest::Entity as OciManifest, oci_tag::Entity as OciTag, oci_upload::Entity as OciUpload, path_check_configs::Entity as PathCheckConfigs, diff --git a/src/ceres/snapshot/error.rs b/src/ceres/snapshot/error.rs index 4fe54274..b3b77296 100644 --- a/src/ceres/snapshot/error.rs +++ b/src/ceres/snapshot/error.rs @@ -18,6 +18,8 @@ pub enum SnapshotErrorCode { ScopeForbidden, ViewNotFound, SnapshotNotReady, + /// A bounded operation could not complete in time (spec 14 §5). + TemporaryUnavailable, /// The fixed view no longer exists: spec 14 §5 SNAPSHOT_GONE (410). SnapshotGone, PathNotFound, @@ -49,6 +51,7 @@ impl SnapshotErrorCode { SnapshotErrorCode::ScopeForbidden => "SCOPE_FORBIDDEN", SnapshotErrorCode::ViewNotFound => "VIEW_NOT_FOUND", SnapshotErrorCode::SnapshotNotReady => "SNAPSHOT_NOT_READY", + SnapshotErrorCode::TemporaryUnavailable => "TEMPORARY_UNAVAILABLE", SnapshotErrorCode::SnapshotGone => "SNAPSHOT_GONE", SnapshotErrorCode::PathNotFound => "PATH_NOT_FOUND", SnapshotErrorCode::NotDirectory => "NOT_DIRECTORY", @@ -82,7 +85,7 @@ impl SnapshotErrorCode { SnapshotErrorCode::ViewNotFound | SnapshotErrorCode::PathNotFound | SnapshotErrorCode::LeaseUnknown => 404, - SnapshotErrorCode::SnapshotNotReady => 503, + SnapshotErrorCode::SnapshotNotReady | SnapshotErrorCode::TemporaryUnavailable => 503, SnapshotErrorCode::ObjectUnavailable => 503, SnapshotErrorCode::NotDirectory | SnapshotErrorCode::Conflict diff --git a/src/jupiter/migration/m20261005_000100_add_mst2_retention_durability.rs b/src/jupiter/migration/m20261005_000100_add_mst2_retention_durability.rs new file mode 100644 index 00000000..8f223af1 --- /dev/null +++ b/src/jupiter/migration/m20261005_000100_add_mst2_retention_durability.rs @@ -0,0 +1,119 @@ +//! T06-B: durable retention counters and replayable GC operations. +//! +//! The original T06 graph migration intentionally supplied only the graph +//! rows. This forward-only migration adds the counter used by the atomic +//! LIVE→DELETING CAS and an idempotent operation log for crash recovery. The +//! runtime is not switched to this schema by this migration alone. + +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Mst2RetentionNode::Table) + .add_column( + big_integer(Mst2RetentionNode::IncomingRefs) + .not_null() + .default(0), + ) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_mst2_retention_node_gc") + .table(Mst2RetentionNode::Table) + .col(Mst2RetentionNode::State) + .col(Mst2RetentionNode::IncomingRefs) + .to_owned(), + ) + .await?; + + manager + .create_table( + Table::create() + .table(Mst2RetentionGcOp::Table) + .if_not_exists() + .col( + string(Mst2RetentionGcOp::OperationId) + .not_null() + .primary_key(), + ) + .col(string(Mst2RetentionGcOp::NodeId).not_null()) + .col(string(Mst2RetentionGcOp::Operation).not_null()) + .col( + string(Mst2RetentionGcOp::State) + .not_null() + .default("PENDING"), + ) + .col(integer(Mst2RetentionGcOp::Attempts).not_null().default(0)) + .col( + timestamp_with_time_zone(Mst2RetentionGcOp::CreatedAt) + .not_null() + .default(Expr::current_timestamp()), + ) + .col(timestamp_with_time_zone_null( + Mst2RetentionGcOp::CompletedAt, + )) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_mst2_retention_gc_op_pending") + .table(Mst2RetentionGcOp::Table) + .col(Mst2RetentionGcOp::State) + .col(Mst2RetentionGcOp::CreatedAt) + .to_owned(), + ) + .await?; + + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_mst2_retention_gc_op_node") + .table(Mst2RetentionGcOp::Table) + .col(Mst2RetentionGcOp::NodeId) + .to_owned(), + ) + .await + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Forward-only: retention state and replay evidence are never + // dropped automatically. + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Mst2RetentionNode { + Table, + State, + IncomingRefs, +} + +#[derive(DeriveIden)] +enum Mst2RetentionGcOp { + Table, + OperationId, + NodeId, + Operation, + State, + Attempts, + CreatedAt, + CompletedAt, +} diff --git a/src/jupiter/migration/m20261005_000200_harden_mst2_retention_graph.rs b/src/jupiter/migration/m20261005_000200_harden_mst2_retention_graph.rs new file mode 100644 index 00000000..c42bcd28 --- /dev/null +++ b/src/jupiter/migration/m20261005_000200_harden_mst2_retention_graph.rs @@ -0,0 +1,76 @@ +//! T06-B: backfill graph counters and reject invalid durable graph state. +//! +//! Applies after the additive schema migration. Existing edges must form a +//! valid DAG and counters are derived from unique edges before any collector +//! can use them. Constraints keep failed writers from silently dangling a +//! node or underflowing a counter. Completed GC receipts intentionally do not +//! reference the removed node through a foreign key. + +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let conn = manager.get_connection(); + conn.execute_unprepared( + "LOCK TABLE mst2_retention_node, mst2_retention_edge, mst2_retention_root, \ + mst2_retention_gc_op IN ACCESS EXCLUSIVE MODE", + ) + .await?; + let cycle = conn + .query_one_raw(sea_orm::Statement::from_string( + sea_orm::DbBackend::Postgres, + "WITH RECURSIVE walk(start_id, node_id) AS ( \ + SELECT parent_id, child_id FROM mst2_retention_edge \ + UNION SELECT w.start_id, e.child_id FROM walk w \ + JOIN mst2_retention_edge e ON e.parent_id = w.node_id \ + ) SELECT start_id FROM walk WHERE start_id = node_id LIMIT 1", + )) + .await?; + if cycle.is_some() { + return Err(DbErr::Migration( + "MST/2 retention graph contains a cycle".into(), + )); + } + conn.execute_unprepared( + "UPDATE mst2_retention_node n SET incoming_refs = \ + (SELECT count(*) FROM mst2_retention_edge e WHERE e.child_id = n.node_id)", + ) + .await?; + conn.execute_unprepared( + "ALTER TABLE mst2_retention_node \ + ADD CONSTRAINT mst2_retention_node_state_check CHECK (state IN ('LIVE', 'DELETING')), \ + ADD CONSTRAINT mst2_retention_node_kind_check \ + CHECK (kind IN ('page', 'chunk_map', 'frame', 'verified_object')), \ + ADD CONSTRAINT mst2_retention_node_nonnegative CHECK (bytes >= 0 AND incoming_refs >= 0); \ + ALTER TABLE mst2_retention_edge \ + ADD CONSTRAINT mst2_retention_edge_parent_fk FOREIGN KEY (parent_id) \ + REFERENCES mst2_retention_node(node_id), \ + ADD CONSTRAINT mst2_retention_edge_child_fk FOREIGN KEY (child_id) \ + REFERENCES mst2_retention_node(node_id), \ + ADD CONSTRAINT mst2_retention_edge_no_self CHECK (parent_id <> child_id); \ + ALTER TABLE mst2_retention_root \ + ADD CONSTRAINT mst2_retention_root_node_fk FOREIGN KEY (node_id) \ + REFERENCES mst2_retention_node(node_id), \ + ADD CONSTRAINT mst2_retention_root_kind_check CHECK (root_kind IN ('lease', 'pin', 'prepare')); \ + ALTER TABLE mst2_retention_gc_op \ + ADD CONSTRAINT mst2_retention_gc_op_kind_check CHECK (operation IN ('MARK_DELETING', 'REMOVE')), \ + ADD CONSTRAINT mst2_retention_gc_op_state_check CHECK (state IN ('PENDING', 'APPLIED', 'FAILED')), \ + ADD CONSTRAINT mst2_retention_gc_op_attempts_check CHECK (attempts >= 0), \ + ADD CONSTRAINT mst2_retention_gc_op_completion_check \ + CHECK ((state = 'APPLIED') = (completed_at IS NOT NULL)); \ + CREATE UNIQUE INDEX idx_mst2_retention_gc_op_pending_node \ + ON mst2_retention_gc_op(node_id) WHERE state = 'PENDING'", + ) + .await + .map(|_| ()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Forward-only: preserve retained nodes and GC recovery evidence. + Ok(()) + } +} diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 39078944..b385d106 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -140,7 +140,9 @@ mod m20260923_000100_import_repo_cleanups; mod m20260923_000200_canonicalize_import_repo_paths; mod m20260925_000100_media_paging; mod m20261005_000100_add_mst2_publication_request_digest; +mod m20261005_000100_add_mst2_retention_durability; mod m20261005_000200_add_mst2_native_head; +mod m20261005_000200_harden_mst2_retention_graph; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -270,6 +272,8 @@ impl MigratorTrait for Migrator { Box::new(m20260925_000100_media_paging::Migration), Box::new(m20261005_000100_add_mst2_publication_request_digest::Migration), Box::new(m20261005_000200_add_mst2_native_head::Migration), + Box::new(m20261005_000100_add_mst2_retention_durability::Migration), + Box::new(m20261005_000200_harden_mst2_retention_graph::Migration), ] } } @@ -479,15 +483,17 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 5..], + &names[names.len() - 7..], &[ "m20260923_000100_import_repo_cleanups".to_string(), "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), "m20261005_000100_add_mst2_publication_request_digest".to_string(), "m20261005_000200_add_mst2_native_head".to_string(), + "m20261005_000100_add_mst2_retention_durability".to_string(), + "m20261005_000200_harden_mst2_retention_graph".to_string(), ], - "native publication registered after media paging and request receipts" + "retention durability registered last, after media paging" ); let db = alias_db().await; diff --git a/src/jupiter/storage/mod.rs b/src/jupiter/storage/mod.rs index 10d4cd63..8ce75f10 100644 --- a/src/jupiter/storage/mod.rs +++ b/src/jupiter/storage/mod.rs @@ -17,6 +17,7 @@ pub mod lfs_db_storage; pub mod media_paging_storage; pub mod mono_storage; pub(crate) mod mst2_publication_storage; +pub mod mst2_retention; pub(crate) mod native_publication_storage; pub mod notification_storage; pub mod object_storage; diff --git a/src/jupiter/storage/mst2_retention.rs b/src/jupiter/storage/mst2_retention.rs new file mode 100644 index 00000000..fe916803 --- /dev/null +++ b/src/jupiter/storage/mst2_retention.rs @@ -0,0 +1,633 @@ +//! Asynchronous PostgreSQL retention graph repository (T06-B, spec 10 §6). +//! +//! This repository is not yet wired into the in-process snapshot runtime. +//! Graph mutations serialize on a schema-scoped transaction advisory lock; +//! callers can include them in a publication transaction. Each mutation uses +//! a savepoint, so a rejected group cannot leave partial data in that caller's +//! READ COMMITTED transaction. Bytes must be complete and verified before +//! retaining a node. +//! +//! A GC claim commits DELETING and a PENDING REMOVE intent together. A worker +//! may repeat the idempotent physical deletion after a crash, then call +//! `complete_gc`. The outgoing edges, child counters and APPLIED receipt commit +//! together, so replay cannot subtract references twice. No method here deletes +//! bytes, and shared Git raw deletion remains disabled until GitRetentionPort +//! is integrated. Physical workers and durable lease/publication wiring remain +//! separate integration work. Removed identities are tombstoned by their GC +//! receipts: rebuilding one requires future generation/fencing integration, +//! otherwise a stale physical worker could delete its newly reconstructed bytes. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use sea_orm::{ + ColumnTrait, ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbBackend, EntityTrait, + QueryFilter, QueryOrder, QuerySelect, Statement, TransactionTrait, +}; +use serde_json::json; + +use crate::{ + callisto::{mst2_retention_gc_op, mst2_retention_node}, + ceres::snapshot::{ + error::{SnapshotError, SnapshotErrorCode}, + retention::{NodeState, RetentionEdge, RetentionNode, RetentionRoot}, + }, +}; + +const MAX_NODES: usize = 4096; +const MAX_EDGES: usize = 16_384; +const MAX_ROOTS: usize = 16; +const MAX_PENDING_BATCH: u64 = 1000; +const RETENTION_LOCK_KEY: i32 = 1_296_717_362; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GcClaim { + /// This transaction marked the node and recorded its pending operation. + Marked, + /// The same operation is committed and awaits physical deletion/ack. + Pending, + /// The same operation already removed the node and released its edges. + Applied, + /// A root, an incoming edge, another claim or absence prevents collection. + Unavailable, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GcCompletion { + pub replayed: bool, + /// Newly zero-reference children; roots must still be checked when claimed. + pub zero_reference_children: Vec, +} + +#[derive(Clone)] +pub struct PostgresRetentionRepository { + connection: DatabaseConnection, +} + +impl PostgresRetentionRepository { + pub fn new(connection: DatabaseConnection) -> Self { + Self { connection } + } + + pub async fn node( + &self, + id: &str, + ) -> Result, SnapshotError> { + mst2_retention_node::Entity::find_by_id(id.to_owned()) + .one(&self.connection) + .await + .map_err(internal) + } + + /// Bounded replay scan. PENDING intents survive process/worker replacement. + pub async fn pending_gc( + &self, + limit: u64, + ) -> Result, SnapshotError> { + if limit == 0 || limit > MAX_PENDING_BATCH { + return Err(limit_error("pending GC batch must be 1..=1000")); + } + mst2_retention_gc_op::Entity::find() + .filter(mst2_retention_gc_op::Column::State.eq("PENDING")) + .order_by_asc(mst2_retention_gc_op::Column::CreatedAt) + .order_by_asc(mst2_retention_gc_op::Column::OperationId) + .limit(limit) + .all(&self.connection) + .await + .map_err(internal) + } + + pub async fn retain_group( + &self, + nodes: &[RetentionNode], + edges: &[RetentionEdge], + roots: &[RetentionRoot], + ) -> Result<(), SnapshotError> { + let txn = self.connection.begin().await.map_err(internal)?; + let result = Self::retain_group_in_txn(&txn, nodes, edges, roots).await; + finish(txn, result).await + } + + /// Retain an immutable group atomically. Roots cover each supplied node; + /// callers select that group rather than scanning the reachable graph. + /// An existing parent's edges may only be replayed, not extended; a new + /// identity describes a new graph. + pub async fn retain_group_in_txn( + txn: &DatabaseTransaction, + nodes: &[RetentionNode], + edges: &[RetentionEdge], + roots: &[RetentionRoot], + ) -> Result<(), SnapshotError> { + let group = PreparedGroup::new(nodes, edges, roots)?; + let savepoint = begin_graph(txn).await?; + let result = retain_locked(&savepoint, &group).await; + finish(savepoint, result).await + } + + pub async fn release_root(&self, root: &RetentionRoot) -> Result<(), SnapshotError> { + let txn = self.connection.begin().await.map_err(internal)?; + let result = Self::release_root_in_txn(&txn, root).await; + finish(txn, result).await + } + + pub async fn release_root_in_txn( + txn: &DatabaseTransaction, + root: &RetentionRoot, + ) -> Result<(), SnapshotError> { + let (key, _) = root_identity(root)?; + let savepoint = begin_graph(txn).await?; + let result = savepoint + .execute_raw(statement( + "DELETE FROM mst2_retention_root WHERE root_key = $1", + [key.into()], + )) + .await + .map(|_| ()) + .map_err(internal); + finish(savepoint, result).await + } + + pub async fn mark_deleting( + &self, + operation_id: &str, + node_id: &str, + ) -> Result { + let txn = self.connection.begin().await.map_err(internal)?; + let result = Self::mark_deleting_in_txn(&txn, operation_id, node_id).await; + finish(txn, result).await + } + + /// The LIVE check, zero-reference check, CAS and replay intent share the + /// retention mutation lock. Acquisition either precedes this CAS or fails. + pub async fn mark_deleting_in_txn( + txn: &DatabaseTransaction, + operation_id: &str, + node_id: &str, + ) -> Result { + validate_id(operation_id)?; + validate_id(node_id)?; + let savepoint = begin_graph(txn).await?; + let result = mark_locked(&savepoint, operation_id, node_id).await; + finish(savepoint, result).await + } + + /// Acknowledge an idempotent, durable physical delete. Until this succeeds + /// DELETING parents retain every child. This changes graph rows only. + pub async fn complete_gc(&self, operation_id: &str) -> Result { + let txn = self.connection.begin().await.map_err(internal)?; + let result = Self::complete_gc_in_txn(&txn, operation_id).await; + finish(txn, result).await + } + + pub async fn complete_gc_in_txn( + txn: &DatabaseTransaction, + operation_id: &str, + ) -> Result { + validate_id(operation_id)?; + let savepoint = begin_graph(txn).await?; + let result = complete_locked(&savepoint, operation_id).await; + finish(savepoint, result).await + } +} + +struct PreparedGroup { + nodes_json: String, + edges_json: String, + roots_json: String, +} + +impl PreparedGroup { + fn new( + nodes: &[RetentionNode], + edges: &[RetentionEdge], + roots: &[RetentionRoot], + ) -> Result { + if nodes.len() > MAX_NODES || edges.len() > MAX_EDGES || roots.len() > MAX_ROOTS { + return Err(limit_error("retention group exceeds bounded batch limits")); + } + let mut unique_nodes = BTreeMap::new(); + for node in nodes { + validate_id(&node.id)?; + if node.state != NodeState::Live { + return Err(unavailable("retention acquisition requires LIVE nodes")); + } + let bytes = i64::try_from(node.bytes) + .map_err(|_| limit_error("retention bytes exceed signed database range"))?; + let definition = (node.kind.as_str(), bytes); + if unique_nodes + .insert(node.id.as_str(), definition) + .is_some_and(|old| old != definition) + { + return Err(integrity("conflicting retention node definitions")); + } + } + let mut unique_edges = BTreeSet::new(); + for edge in edges { + validate_id(&edge.parent)?; + validate_id(&edge.child)?; + unique_edges.insert((edge.parent.as_str(), edge.child.as_str())); + } + reject_cycle(&unique_edges)?; + let roots: BTreeSet<_> = roots.iter().map(root_identity).collect::>()?; + Ok(Self { + nodes_json: json!( + unique_nodes + .iter() + .map(|(id, (kind, bytes))| { + json!({"node_id": id, "kind": kind, "bytes": bytes}) + }) + .collect::>() + ) + .to_string(), + edges_json: json!( + unique_edges + .iter() + .map(|(parent, child)| { json!({"parent_id": parent, "child_id": child}) }) + .collect::>() + ) + .to_string(), + roots_json: json!( + roots + .iter() + .map(|(key, kind)| { json!({"root_key": key, "root_kind": kind}) }) + .collect::>() + ) + .to_string(), + }) + } +} + +fn reject_cycle(edges: &BTreeSet<(&str, &str)>) -> Result<(), SnapshotError> { + let mut incoming: BTreeMap<&str, usize> = BTreeMap::new(); + let mut children: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for &(parent, child) in edges { + incoming.entry(parent).or_default(); + *incoming.entry(child).or_default() += 1; + children.entry(parent).or_default().push(child); + } + let mut ready: VecDeque<_> = incoming + .iter() + .filter_map(|(&id, &count)| (count == 0).then_some(id)) + .collect(); + let mut processed = 0; + while let Some(id) = ready.pop_front() { + processed += 1; + if let Some(children) = children.get(id) { + for child in children { + if let Some(count) = incoming.get_mut(child) { + *count -= 1; + if *count == 0 { + ready.push_back(child); + } + } + } + } + } + if processed != incoming.len() { + return Err(integrity("retention graph must be acyclic")); + } + Ok(()) +} + +async fn retain_locked( + txn: &DatabaseTransaction, + group: &PreparedGroup, +) -> Result<(), SnapshotError> { + if txn + .query_one_raw(statement( + "SELECT i.node_id FROM jsonb_to_recordset($1::jsonb) AS i(node_id text) \ + WHERE NOT EXISTS (SELECT 1 FROM mst2_retention_node n WHERE n.node_id = i.node_id) \ + AND EXISTS (SELECT 1 FROM mst2_retention_gc_op op WHERE op.node_id = i.node_id) LIMIT 1", + [group.nodes_json.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(unavailable( + "removed node identity requires generation-fenced reconstruction", + )); + } + // Batch comparisons happen before any externally visible commit. An + // identity never changes kind, size or DELETING state on replay. + if txn + .query_one_raw(statement( + "SELECT n.node_id, n.state FROM mst2_retention_node n \ + JOIN jsonb_to_recordset($1::jsonb) AS i(node_id text, kind text, bytes bigint) \ + ON n.node_id = i.node_id \ + WHERE n.state <> 'LIVE' OR n.kind <> i.kind OR n.bytes <> i.bytes LIMIT 1", + [group.nodes_json.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(unavailable( + "existing node is DELETING or conflicts with immutable identity", + )); + } + if txn + .query_one_raw(statement( + "SELECT e.parent_id FROM \ + jsonb_to_recordset($1::jsonb) AS e(parent_id text, child_id text) \ + JOIN mst2_retention_node n ON n.node_id = e.parent_id \ + WHERE NOT EXISTS (SELECT 1 FROM mst2_retention_edge old \ + WHERE old.parent_id = e.parent_id AND old.child_id = e.child_id) LIMIT 1", + [group.edges_json.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(integrity( + "an existing retention parent's edges are immutable", + )); + } + txn.execute_raw(statement( + "INSERT INTO mst2_retention_node (node_id, kind, state, bytes, incoming_refs, created_at) \ + SELECT node_id, kind, 'LIVE', bytes, 0, now() FROM \ + jsonb_to_recordset($1::jsonb) AS i(node_id text, kind text, bytes bigint) \ + ON CONFLICT (node_id) DO NOTHING", + [group.nodes_json.clone().into()], + )) + .await + .map_err(internal)?; + if txn + .query_one_raw(statement( + "SELECT e.parent_id FROM \ + jsonb_to_recordset($1::jsonb) AS e(parent_id text, child_id text) \ + LEFT JOIN mst2_retention_node p ON p.node_id = e.parent_id \ + LEFT JOIN mst2_retention_node c ON c.node_id = e.child_id \ + WHERE p.node_id IS NULL OR c.node_id IS NULL \ + OR p.state <> 'LIVE' OR c.state <> 'LIVE' LIMIT 1", + [group.edges_json.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(unavailable( + "retention edge requires two known LIVE endpoints", + )); + } + txn.execute_raw(statement( + "WITH added AS ( \ + INSERT INTO mst2_retention_edge (parent_id, child_id, created_at) \ + SELECT parent_id, child_id, now() FROM \ + jsonb_to_recordset($1::jsonb) AS e(parent_id text, child_id text) \ + ON CONFLICT (parent_id, child_id) DO NOTHING RETURNING child_id \ + ), delta AS (SELECT child_id, count(*) AS refs FROM added GROUP BY child_id) \ + UPDATE mst2_retention_node n SET incoming_refs = n.incoming_refs + delta.refs \ + FROM delta WHERE n.node_id = delta.child_id", + [group.edges_json.clone().into()], + )) + .await + .map_err(internal)?; + txn.execute_raw(statement( + "INSERT INTO mst2_retention_root (node_id, root_key, root_kind, created_at) \ + SELECT n.node_id, r.root_key, r.root_kind, now() FROM \ + jsonb_to_recordset($1::jsonb) AS n(node_id text) CROSS JOIN \ + jsonb_to_recordset($2::jsonb) AS r(root_key text, root_kind text) \ + ON CONFLICT (node_id, root_key) DO NOTHING", + [ + group.nodes_json.clone().into(), + group.roots_json.clone().into(), + ], + )) + .await + .map_err(internal)?; + Ok(()) +} + +async fn mark_locked( + txn: &DatabaseTransaction, + operation_id: &str, + node_id: &str, +) -> Result { + if let Some(op) = mst2_retention_gc_op::Entity::find_by_id(operation_id.to_owned()) + .one(txn) + .await + .map_err(internal)? + { + if op.node_id != node_id || op.operation != "REMOVE" { + return Err(integrity("GC operation id is bound to different work")); + } + return match op.state.as_str() { + "PENDING" => Ok(GcClaim::Pending), + "APPLIED" => Ok(GcClaim::Applied), + _ => Err(integrity("GC operation is not replayable")), + }; + } + let row = txn + .query_one_raw(statement( + "SELECT n.state, n.incoming_refs, \ + (SELECT count(*) FROM mst2_retention_edge e WHERE e.child_id = n.node_id) AS actual_refs \ + FROM mst2_retention_node n WHERE n.node_id = $1", + [node_id.into()], + )) + .await + .map_err(internal)?; + let Some(row) = row else { + return Ok(GcClaim::Unavailable); + }; + let refs: i64 = row.try_get("", "incoming_refs").map_err(internal)?; + let actual: i64 = row.try_get("", "actual_refs").map_err(internal)?; + if refs != actual || refs < 0 { + return Err(integrity("retention reference audit failed; GC stopped")); + } + if row.try_get::("", "state").map_err(internal)? != "LIVE" || refs != 0 { + return Ok(GcClaim::Unavailable); + } + let changed = txn + .execute_raw(statement( + "UPDATE mst2_retention_node n SET state = 'DELETING' \ + WHERE node_id = $1 AND state = 'LIVE' AND incoming_refs = 0 \ + AND NOT EXISTS (SELECT 1 FROM mst2_retention_root r WHERE r.node_id = n.node_id) \ + AND NOT EXISTS (SELECT 1 FROM mst2_retention_edge e WHERE e.child_id = n.node_id)", + [node_id.into()], + )) + .await + .map_err(internal)?; + if changed.rows_affected() == 0 { + return Ok(GcClaim::Unavailable); + } + txn.execute_raw(statement( + "INSERT INTO mst2_retention_gc_op (operation_id, node_id, operation, state, attempts, created_at) \ + VALUES ($1, $2, 'REMOVE', 'PENDING', 0, now())", + [operation_id.into(), node_id.into()], + )) + .await + .map_err(internal)?; + Ok(GcClaim::Marked) +} + +async fn complete_locked( + txn: &DatabaseTransaction, + operation_id: &str, +) -> Result { + let op = mst2_retention_gc_op::Entity::find_by_id(operation_id.to_owned()) + .one(txn) + .await + .map_err(internal)? + .ok_or_else(|| integrity("unknown GC operation"))?; + if op.operation != "REMOVE" { + return Err(integrity("GC operation is not a removal")); + } + if op.state == "APPLIED" { + return Ok(GcCompletion { + replayed: true, + zero_reference_children: Vec::new(), + }); + } + if op.state != "PENDING" { + return Err(integrity("GC operation is not replayable")); + } + let node = mst2_retention_node::Entity::find_by_id(op.node_id.clone()) + .one(txn) + .await + .map_err(internal)? + .ok_or_else(|| integrity("pending GC node is missing"))?; + if node.state != "DELETING" || node.incoming_refs != 0 { + return Err(integrity("pending GC node is not unreferenced DELETING")); + } + if txn + .query_one_raw(statement( + "SELECT node_id FROM mst2_retention_root WHERE node_id = $1 \ + UNION ALL SELECT child_id FROM mst2_retention_edge WHERE child_id = $1 LIMIT 1", + [op.node_id.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(integrity("pending GC node gained a reference; GC stopped")); + } + // Audit before subtracting. Even a damaged stored counter must not cause + // a child's data to be treated as unreferenced. + if txn.query_one_raw(statement( + "SELECT c.node_id FROM mst2_retention_edge e \ + JOIN mst2_retention_node c ON c.node_id = e.child_id \ + WHERE e.parent_id = $1 AND (c.state <> 'LIVE' OR c.incoming_refs <= 0 OR \ + c.incoming_refs <> (SELECT count(*) FROM mst2_retention_edge i WHERE i.child_id = c.node_id)) \ + LIMIT 1", + [op.node_id.clone().into()], + )).await.map_err(internal)?.is_some() { + return Err(integrity("child reference audit failed; GC stopped")); + } + let rows = txn.query_all_raw(statement( + "WITH removed AS (DELETE FROM mst2_retention_edge WHERE parent_id = $1 RETURNING child_id), \ + delta AS (SELECT child_id, count(*) AS refs FROM removed GROUP BY child_id) \ + UPDATE mst2_retention_node n SET incoming_refs = n.incoming_refs - delta.refs \ + FROM delta WHERE n.node_id = delta.child_id RETURNING n.node_id, n.incoming_refs", + [op.node_id.clone().into()], + )).await.map_err(internal)?; + let mut zero_reference_children = Vec::new(); + for row in rows { + if row.try_get::("", "incoming_refs").map_err(internal)? == 0 { + zero_reference_children.push(row.try_get("", "node_id").map_err(internal)?); + } + } + zero_reference_children.sort(); + txn.execute_raw(statement( + "DELETE FROM mst2_retention_node WHERE node_id = $1 AND state = 'DELETING'", + [op.node_id.into()], + )) + .await + .map_err(internal)?; + txn.execute_raw(statement( + "UPDATE mst2_retention_gc_op SET state = 'APPLIED', completed_at = now(), attempts = attempts + 1 \ + WHERE operation_id = $1 AND state = 'PENDING'", + [operation_id.into()], + )).await.map_err(internal)?; + Ok(GcCompletion { + replayed: false, + zero_reference_children, + }) +} + +async fn begin_graph(txn: &DatabaseTransaction) -> Result { + if txn.get_database_backend() != DbBackend::Postgres { + return Err(internal("retention repository requires PostgreSQL")); + } + let isolation = txn + .query_one_raw(statement("SHOW transaction_isolation", [])) + .await + .map_err(internal)? + .ok_or_else(|| internal("transaction isolation query returned no row"))? + .try_get_by_index::(0) + .map_err(internal)?; + if isolation != "read committed" { + return Err(internal( + "retention mutations require READ COMMITTED transaction isolation", + )); + } + let savepoint = txn.begin().await.map_err(internal)?; + let result = savepoint + .execute_raw(statement( + "SELECT pg_advisory_xact_lock($1, hashtext(current_schema()))", + [RETENTION_LOCK_KEY.into()], + )) + .await + .map_err(internal); + if let Err(err) = result { + return finish(savepoint, Err(err)).await; + } + Ok(savepoint) +} + +async fn finish( + txn: DatabaseTransaction, + result: Result, +) -> Result { + match result { + Ok(value) => { + txn.commit().await.map_err(internal)?; + Ok(value) + } + Err(err) => { + txn.rollback().await.map_err(internal)?; + Err(err) + } + } +} + +fn statement(sql: &str, values: [sea_orm::Value; N]) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, sql, values) +} + +fn validate_id(id: &str) -> Result<(), SnapshotError> { + if id.is_empty() || id.len() > 255 || id.contains('\0') { + return Err(limit_error( + "retention identifiers require 1..=255 bytes without NUL", + )); + } + Ok(()) +} + +fn root_identity(root: &RetentionRoot) -> Result<(String, &'static str), SnapshotError> { + let (id, kind) = match root { + RetentionRoot::Lease(id) => (id, "lease"), + RetentionRoot::Pin(id) => (id, "pin"), + RetentionRoot::Prepare(id) => (id, "prepare"), + }; + validate_id(id)?; + let key = format!("{kind}:{id}"); + validate_id(&key)?; + Ok((key, kind)) +} + +fn internal(error: impl std::fmt::Display) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::Internal, error.to_string()) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} +fn unavailable(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::ObjectUnavailable, message) +} +fn limit_error(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::LimitExceeded, message) +} + +#[cfg(test)] +#[path = "mst2_retention_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/mst2_retention_tests.rs b/src/jupiter/storage/mst2_retention_tests.rs new file mode 100644 index 00000000..a8f7c866 --- /dev/null +++ b/src/jupiter/storage/mst2_retention_tests.rs @@ -0,0 +1,629 @@ +use std::time::Duration; + +use sea_orm::{ + ConnectionTrait, DatabaseConnection, EntityTrait, IsolationLevel, PaginatorTrait, + TransactionTrait, +}; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::{ + callisto::{mst2_retention_edge, mst2_retention_root}, + ceres::snapshot::retention::RetainedKind, + jupiter::{migration::Migrator, tests::test_db_connection}, +}; + +fn node(id: &str) -> RetentionNode { + RetentionNode { + id: id.into(), + kind: RetainedKind::Page, + state: NodeState::Live, + bytes: 7, + } +} + +fn edge(parent: &str, child: &str) -> RetentionEdge { + RetentionEdge { + parent: parent.into(), + child: child.into(), + } +} + +async fn fixture() -> (DatabaseConnection, PostgresRetentionRepository) { + let temp = tempfile::TempDir::new().expect("temp directory"); + let db = test_db_connection(temp.path()).await; + Migrator::up(&db, None) + .await + .expect("isolated schema migrations"); + let repository = PostgresRetentionRepository::new(db.clone()); + (db, repository) +} + +#[test] +fn t06b_rejects_cycles_conflicting_identities_and_unbounded_groups() { + let a = node("a"); + let b = node("b"); + assert_eq!( + PreparedGroup::new(&[a.clone(), b], &[edge("a", "b"), edge("b", "a")], &[]) + .err() + .expect("cycle rejected") + .code, + SnapshotErrorCode::IntegrityError + ); + let mut conflicting = a.clone(); + conflicting.bytes += 1; + assert!(PreparedGroup::new(&[a.clone(), conflicting], &[], &[]).is_err()); + assert_eq!( + PreparedGroup::new(&vec![a; MAX_NODES + 1], &[], &[]) + .err() + .expect("batch rejected") + .code, + SnapshotErrorCode::LimitExceeded + ); +} + +#[tokio::test] +async fn t06b_retains_unique_edges_and_independent_roots_once() { + let (db, repository) = fixture().await; + let roots = [ + RetentionRoot::Lease("one".into()), + RetentionRoot::Pin("two".into()), + ]; + let nodes = [node("parent"), node("child")]; + let edges = [edge("parent", "child"), edge("parent", "child")]; + for _ in 0..2 { + repository + .retain_group(&nodes, &edges, &roots) + .await + .expect("idempotent retain"); + } + assert_eq!( + repository + .node("child") + .await + .unwrap() + .unwrap() + .incoming_refs, + 1 + ); + assert_eq!( + mst2_retention_edge::Entity::find() + .count(&db) + .await + .unwrap(), + 1 + ); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 4 + ); + repository.release_root(&roots[0]).await.unwrap(); + repository.release_root(&roots[0]).await.unwrap(); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 2 + ); + assert_eq!( + repository + .mark_deleting("gc-parent", "parent") + .await + .unwrap(), + GcClaim::Unavailable + ); + repository.release_root(&roots[1]).await.unwrap(); + assert_eq!( + repository + .mark_deleting("gc-parent", "parent") + .await + .unwrap(), + GcClaim::Marked + ); + assert_eq!( + repository.mark_deleting("gc-child", "child").await.unwrap(), + GcClaim::Unavailable + ); +} + +#[tokio::test] +async fn t06b_group_failure_rolls_back_savepoint_even_if_caller_commits() { + let (db, repository) = fixture().await; + repository + .retain_group(&[node("old")], &[], &[]) + .await + .unwrap(); + assert_eq!( + repository.mark_deleting("old-gc", "old").await.unwrap(), + GcClaim::Marked + ); + let txn = db.begin().await.unwrap(); + let error = PostgresRetentionRepository::retain_group_in_txn( + &txn, + &[node("new")], + &[edge("new", "old")], + &[RetentionRoot::Lease("partial".into())], + ) + .await + .expect_err("DELETING child rejects whole group"); + assert_eq!(error.code, SnapshotErrorCode::ObjectUnavailable); + txn.commit().await.unwrap(); + assert!(repository.node("new").await.unwrap().is_none()); + assert_eq!( + repository.node("old").await.unwrap().unwrap().state, + "DELETING" + ); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); + assert_eq!( + mst2_retention_edge::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn t06b_publication_rollback_removes_retention_acquisition() { + let (db, repository) = fixture().await; + let txn = db.begin().await.unwrap(); + PostgresRetentionRepository::retain_group_in_txn( + &txn, + &[node("publication")], + &[], + &[RetentionRoot::Prepare("writer".into())], + ) + .await + .unwrap(); + txn.rollback().await.unwrap(); + assert!(repository.node("publication").await.unwrap().is_none()); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn t06b_root_release_rollback_preserves_coverage() { + let (db, repository) = fixture().await; + let root = RetentionRoot::Lease("reader".into()); + repository + .retain_group(&[node("page")], &[], std::slice::from_ref(&root)) + .await + .unwrap(); + let txn = db.begin().await.unwrap(); + PostgresRetentionRepository::release_root_in_txn(&txn, &root) + .await + .unwrap(); + txn.rollback().await.unwrap(); + assert_eq!( + repository.mark_deleting("page-op", "page").await.unwrap(), + GcClaim::Unavailable + ); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 1 + ); +} + +#[tokio::test] +async fn t06b_rejects_snapshot_isolation_without_mutating_outer_transaction() { + let (db, repository) = fixture().await; + let txn = db + .begin_with_config(Some(IsolationLevel::RepeatableRead), None) + .await + .unwrap(); + let error = PostgresRetentionRepository::retain_group_in_txn( + &txn, + &[node("page")], + &[], + &[RetentionRoot::Lease("reader".into())], + ) + .await + .expect_err("old MVCC snapshots cannot safely follow an advisory lock"); + assert_eq!(error.code, SnapshotErrorCode::Internal); + assert!(error.message.contains("READ COMMITTED")); + txn.commit().await.unwrap(); + assert!(repository.node("page").await.unwrap().is_none()); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn t06b_existing_parent_and_node_definition_are_immutable() { + let (db, repository) = fixture().await; + repository + .retain_group(&[node("parent"), node("child")], &[], &[]) + .await + .unwrap(); + let mut changed = node("parent"); + changed.bytes += 1; + assert!(repository.retain_group(&[changed], &[], &[]).await.is_err()); + assert!( + repository + .retain_group(&[node("parent")], &[edge("parent", "child")], &[]) + .await + .is_err() + ); + assert_eq!(repository.node("parent").await.unwrap().unwrap().bytes, 7); + assert_eq!( + mst2_retention_edge::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); +} + +#[tokio::test] +async fn t06b_gc_replay_subtracts_edges_once_and_tombstones_removed_identity() { + let (db, repository) = fixture().await; + repository + .retain_group( + &[node("parent"), node("child")], + &[edge("parent", "child")], + &[], + ) + .await + .unwrap(); + assert_eq!( + repository + .mark_deleting("parent-op", "parent") + .await + .unwrap(), + GcClaim::Marked + ); + let restarted = PostgresRetentionRepository::new(db.clone()); + let pending = restarted.pending_gc(10).await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].operation_id, "parent-op"); + assert_eq!( + restarted + .mark_deleting("parent-op", "parent") + .await + .unwrap(), + GcClaim::Pending + ); + assert_eq!( + restarted + .mark_deleting("another-op", "parent") + .await + .unwrap(), + GcClaim::Unavailable + ); + assert!( + restarted + .mark_deleting("parent-op", "different") + .await + .is_err() + ); + + // Simulated crash/rollback during bookkeeping: the PENDING intent and + // child's reference survive and can be replayed after reaper replacement. + let txn = db.begin().await.unwrap(); + let provisional = PostgresRetentionRepository::complete_gc_in_txn(&txn, "parent-op") + .await + .unwrap(); + assert_eq!(provisional.zero_reference_children, vec!["child"]); + txn.rollback().await.unwrap(); + assert_eq!( + restarted + .node("child") + .await + .unwrap() + .unwrap() + .incoming_refs, + 1 + ); + assert_eq!(restarted.pending_gc(10).await.unwrap().len(), 1); + assert_eq!( + restarted.node("parent").await.unwrap().unwrap().state, + "DELETING" + ); + + let completed = restarted.complete_gc("parent-op").await.unwrap(); + assert!(!completed.replayed); + assert_eq!(completed.zero_reference_children, vec!["child"]); + assert!(restarted.node("parent").await.unwrap().is_none()); + assert_eq!( + restarted + .node("child") + .await + .unwrap() + .unwrap() + .incoming_refs, + 0 + ); + assert!(restarted.complete_gc("parent-op").await.unwrap().replayed); + assert_eq!( + restarted + .mark_deleting("parent-op", "parent") + .await + .unwrap(), + GcClaim::Applied + ); + assert!(restarted.pending_gc(10).await.unwrap().is_empty()); + let receipt = mst2_retention_gc_op::Entity::find_by_id("parent-op".to_owned()) + .one(&db) + .await + .unwrap() + .unwrap(); + assert_eq!(receipt.attempts, 1); + assert!(receipt.completed_at.is_some()); + + // Old pending workers must not be able to delete same-id reconstructed + // bytes. Reconstruction requires a future generation/fencing protocol. + assert_eq!( + restarted + .retain_group(&[node("parent")], &[], &[]) + .await + .unwrap_err() + .code, + SnapshotErrorCode::ObjectUnavailable + ); + assert!(restarted.complete_gc("parent-op").await.unwrap().replayed); + assert!(restarted.node("parent").await.unwrap().is_none()); +} + +#[tokio::test] +async fn t06b_counter_audit_stops_gc_and_preserves_pending_edges() { + let (db, repository) = fixture().await; + repository + .retain_group( + &[node("parent"), node("child")], + &[edge("parent", "child")], + &[], + ) + .await + .unwrap(); + db.execute_unprepared( + "UPDATE mst2_retention_node SET incoming_refs = 0 WHERE node_id = 'child'", + ) + .await + .unwrap(); + assert_eq!( + repository + .mark_deleting("child-op", "child") + .await + .unwrap_err() + .code, + SnapshotErrorCode::IntegrityError + ); + assert_eq!( + repository + .mark_deleting("parent-op", "parent") + .await + .unwrap(), + GcClaim::Marked + ); + assert_eq!( + repository.complete_gc("parent-op").await.unwrap_err().code, + SnapshotErrorCode::IntegrityError + ); + assert_eq!( + mst2_retention_edge::Entity::find() + .count(&db) + .await + .unwrap(), + 1 + ); + assert_eq!(repository.pending_gc(10).await.unwrap().len(), 1); + assert_eq!( + repository.node("child").await.unwrap().unwrap().state, + "LIVE" + ); + db.execute_unprepared( + "UPDATE mst2_retention_node SET incoming_refs = 1 WHERE node_id = 'child'", + ) + .await + .unwrap(); + repository.complete_gc("parent-op").await.unwrap(); + assert_eq!( + repository + .node("child") + .await + .unwrap() + .unwrap() + .incoming_refs, + 0 + ); +} + +#[tokio::test] +async fn t06b_root_acquisition_wins_before_gc_cas() { + let (db, repository) = fixture().await; + repository + .retain_group(&[node("page")], &[], &[]) + .await + .unwrap(); + let root_txn = db.begin().await.unwrap(); + PostgresRetentionRepository::retain_group_in_txn( + &root_txn, + &[node("page")], + &[], + &[RetentionRoot::Lease("winner".into())], + ) + .await + .unwrap(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let gc_repository = repository.clone(); + let mut gc = tokio::spawn(async move { + started_tx.send(()).unwrap(); + gc_repository.mark_deleting("gc-op", "page").await + }); + started_rx.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut gc) + .await + .is_err() + ); + root_txn.commit().await.unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), gc) + .await + .unwrap() + .unwrap() + .unwrap(), + GcClaim::Unavailable + ); + assert_eq!( + repository.node("page").await.unwrap().unwrap().state, + "LIVE" + ); + assert!(repository.pending_gc(10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn t06b_gc_cas_wins_and_new_root_cannot_reference_deleting_node() { + let (db, repository) = fixture().await; + repository + .retain_group(&[node("page")], &[], &[]) + .await + .unwrap(); + let gc_txn = db.begin().await.unwrap(); + assert_eq!( + PostgresRetentionRepository::mark_deleting_in_txn(&gc_txn, "gc-op", "page") + .await + .unwrap(), + GcClaim::Marked + ); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let acquire_repository = repository.clone(); + let mut acquire = tokio::spawn(async move { + started_tx.send(()).unwrap(); + acquire_repository + .retain_group(&[node("page")], &[], &[RetentionRoot::Lease("late".into())]) + .await + }); + started_rx.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut acquire) + .await + .is_err() + ); + gc_txn.commit().await.unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), acquire) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code, + SnapshotErrorCode::ObjectUnavailable + ); + assert_eq!( + mst2_retention_root::Entity::find() + .count(&db) + .await + .unwrap(), + 0 + ); + assert_eq!(repository.pending_gc(10).await.unwrap().len(), 1); +} + +#[tokio::test] +async fn t06b_forward_migration_backfills_old_graph_and_enforces_constraints() { + let temp = tempfile::TempDir::new().unwrap(); + let db = test_db_connection(temp.path()).await; + let at = Migrator::migrations() + .iter() + .position(|migration| migration.name() == "m20261005_000200_harden_mst2_retention_graph") + .expect("hardening migration registered"); + Migrator::up(&db, Some(at.try_into().unwrap())) + .await + .unwrap(); + db.execute_unprepared( + "INSERT INTO mst2_retention_node (node_id, kind, state, bytes, created_at) \ + VALUES ('parent', 'page', 'LIVE', 7, now()), ('child', 'page', 'LIVE', 7, now()); \ + INSERT INTO mst2_retention_edge (parent_id, child_id, created_at) VALUES ('parent', 'child', now())", + ) + .await + .unwrap(); + Migrator::up(&db, None).await.unwrap(); + let repository = PostgresRetentionRepository::new(db.clone()); + assert_eq!( + repository + .node("child") + .await + .unwrap() + .unwrap() + .incoming_refs, + 1 + ); + assert!( + db.execute_unprepared( + "INSERT INTO mst2_retention_edge (parent_id, child_id, created_at) VALUES ('missing', 'child', now())" + ) + .await + .is_err() + ); + assert!( + db.execute_unprepared( + "UPDATE mst2_retention_node SET incoming_refs = -1 WHERE node_id = 'child'" + ) + .await + .is_err() + ); + let completion_error = db + .execute_unprepared( + "INSERT INTO mst2_retention_gc_op (operation_id, node_id, operation, state, attempts, created_at) \ + VALUES ('bad', 'child', 'REMOVE', 'APPLIED', 0, now())", + ) + .await + .expect_err("APPLIED receipt without completed_at must violate its completion constraint"); + assert!( + completion_error + .to_string() + .contains("mst2_retention_gc_op_completion_check"), + "unexpected constraint failure: {completion_error}" + ); +} + +#[tokio::test] +async fn t06b_forward_migration_refuses_existing_cycles() { + let temp = tempfile::TempDir::new().unwrap(); + let db = test_db_connection(temp.path()).await; + let at = Migrator::migrations() + .iter() + .position(|migration| migration.name() == "m20261005_000200_harden_mst2_retention_graph") + .unwrap(); + Migrator::up(&db, Some(at.try_into().unwrap())) + .await + .unwrap(); + db.execute_unprepared( + "INSERT INTO mst2_retention_node (node_id, kind, state, bytes, created_at) \ + VALUES ('a', 'page', 'LIVE', 7, now()), ('b', 'page', 'LIVE', 7, now()); \ + INSERT INTO mst2_retention_edge (parent_id, child_id, created_at) VALUES ('a', 'b', now()), ('b', 'a', now())", + ) + .await + .unwrap(); + assert!(Migrator::up(&db, None).await.is_err()); + assert_eq!( + mst2_retention_edge::Entity::find() + .count(&db) + .await + .unwrap(), + 2 + ); +} From 17be89d204e3d3dbce0d6a94550f2fa579aff794 Mon Sep 17 00:00:00 2001 From: Luxian Date: Tue, 6 Oct 2026 02:19:23 +0800 Subject: [PATCH 14/14] Require the owned HTTP child to hold its readiness listener --- tests/integration_api_write_trunk.rs | 68 ++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/tests/integration_api_write_trunk.rs b/tests/integration_api_write_trunk.rs index 569340ce..48e710d6 100644 --- a/tests/integration_api_write_trunk.rs +++ b/tests/integration_api_write_trunk.rs @@ -232,6 +232,63 @@ struct ServiceProcess { reaped: bool, } +#[cfg(target_os = "linux")] +fn process_owns_tcp_listener(pid: u32, port: u16) -> std::io::Result { + let proc = PathBuf::from(format!("/proc/{pid}")); + let sockets: std::collections::HashSet<_> = fs::read_dir(proc.join("fd"))? + .filter_map(Result::ok) + .filter_map(|entry| fs::read_link(entry.path()).ok()) + .filter_map(|target| { + target + .to_str()? + .strip_prefix("socket:[")? + .strip_suffix(']') + .map(str::to_owned) + }) + .collect(); + for table in ["tcp", "tcp6"] { + let rows = fs::read_to_string(proc.join("net").join(table))?; + for row in rows.lines().skip(1) { + let fields: Vec<_> = row.split_whitespace().collect(); + if fields.len() < 10 || fields[3] != "0A" || !sockets.contains(fields[9]) { + continue; + } + let Some((address, encoded_port)) = fields[1].split_once(':') else { + continue; + }; + if u16::from_str_radix(encoded_port, 16).ok() == Some(port) + && matches!( + address, + "00000000" | "0100007F" | "00000000000000000000000000000000" + ) + { + return Ok(true); + } + } + } + Ok(false) +} + +#[cfg(target_os = "linux")] +#[test] +fn service_readiness_accepts_its_own_kernel_listener() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + assert!(process_owns_tcp_listener(std::process::id(), port).unwrap()); +} + +#[cfg(target_os = "linux")] +#[test] +fn service_readiness_rejects_a_connectable_foreign_listener() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut command = Command::new("sh"); + command.args(["-c", "exec sleep 60"]); + let service = ServiceProcess::spawn(command); + assert!(TcpStream::connect(("127.0.0.1", port)).is_ok()); + assert!(!process_owns_tcp_listener(service.child.id(), port).unwrap()); +} + impl ServiceProcess { fn spawn(mut command: Command) -> Self { let child = command.spawn().expect("spawn mega2 service"); @@ -251,9 +308,6 @@ impl ServiceProcess { ) { let deadline = Instant::now() + timeout; loop { - if TcpStream::connect(("127.0.0.1", port)).is_ok() { - return; - } if let Some(status) = self.child.try_wait().expect("poll service") { self.reaped = true; panic!( @@ -262,6 +316,14 @@ impl ServiceProcess { read_log(stderr_path), ); } + // Another test's short-lived socket cannot establish this child's readiness. + #[cfg(target_os = "linux")] + let owns_listener = process_owns_tcp_listener(self.child.id(), port).unwrap_or(false); + #[cfg(not(target_os = "linux"))] + let owns_listener = true; + if owns_listener && TcpStream::connect(("127.0.0.1", port)).is_ok() { + return; + } if Instant::now() >= deadline { panic!( "service did not bind port {port} within {timeout:?}\nstdout:\n{}\nstderr:\n{}",