diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e79009d..11ce3d72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,18 @@ jobs: - name: Run tests run: cargo test --verbose + - name: Check real runner budgets and process cleanup + run: | + python3 -B tests/mst2-e2e/test_commit_update_bench.py -v + python3 -B tests/mst2-e2e/test_commit_update_budget.py -v + python3 -B tests/mst2-e2e/test_commit_update_projection.py -v + + - name: Check measurement driver old-view audit + run: | + cargo build --locked --release --example mst2_update_measure + MST2_MEASURE_DRIVER="$GITHUB_WORKSPACE/target/release/examples/mst2_update_measure" \ + cargo test --locked --release --example mst2_update_measure + docker: name: Docker image runs-on: ubuntu-latest diff --git a/.github/workflows/mst2-projection-update.yml b/.github/workflows/mst2-projection-update.yml new file mode 100644 index 00000000..a66cc7c4 --- /dev/null +++ b/.github/workflows/mst2-projection-update.yml @@ -0,0 +1,132 @@ +name: MST2 request-bound projection update benchmark + +on: + workflow_dispatch: + inputs: + mega_sha: + description: Immutable reviewed server SHA with typed projection writer + required: true + profile: + type: choice + description: Fixture size + options: [medium, smoke] + default: medium + rounds: + description: Complete repetitions (3 to 10) + required: true + default: '3' + session_started_utc: + description: Actual dispatch UTC T0, retained on recovery + required: true + session_deadline_utc: + description: Original absolute UTC D, exactly T0 plus 235 minutes + required: true + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 220 + env: + CARGO_BUILD_JOBS: '2' + CARGO_INCREMENTAL: '0' + CARGO_PROFILE_RELEASE_DEBUG: '0' + NO_PROXY: localhost,127.0.0.1 + MEGA_SHA: ${{ inputs.mega_sha }} + SCORPIO_SHA: ${{ github.sha }} + PROFILE: ${{ inputs.profile }} + ROUNDS: ${{ inputs.rounds }} + STARTED_INPUT: ${{ inputs.session_started_utc }} + DEADLINE_INPUT: ${{ inputs.session_deadline_utc }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ env.SCORPIO_SHA }} + path: scorpiofs + persist-credentials: false + + - name: Admit original dispatch window before builds + shell: bash + run: | + python3 -B - <<'PY' + import os, sys + sys.path.insert(0, 'scorpiofs/tests/mst2-e2e') + from commit_update_projection import window_anchor + anchor = window_anchor(os.environ['STARTED_INPUT'], os.environ['DEADLINE_INPUT'], admission=True) + with open(os.environ['GITHUB_ENV'], 'a') as stream: + stream.write('MST2_SESSION_STARTED=' + os.environ['STARTED_INPUT'] + '\n') + stream.write('MST2_SESSION_DEADLINE=' + os.environ['DEADLINE_INPUT'] + '\n') + stream.write('MST2_WORK_CLEANUP_DEADLINE_MONOTONIC=' + repr(anchor) + '\n') + stream.write('MST2_OWNED_ROOT=' + os.environ['RUNNER_TEMP'] + '/mst2-real-' + os.environ['GITHUB_RUN_ID'] + '-' + os.environ['GITHUB_RUN_ATTEMPT'] + '\n') + PY + + - uses: actions/checkout@v7 + with: + repository: gitmono-dev/mega2 + ref: ${{ env.MEGA_SHA }} + path: mega2 + persist-credentials: false + + - uses: actions/cache@v6 + with: + path: scorpiofs/target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('scorpiofs/Cargo.lock') }} + + - name: Install system dependencies + timeout-minutes: 10 + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev libfuse3-dev protobuf-compiler postgresql-client + + - uses: dtolnay/rust-toolchain@stable + + - name: Build immutable server and measurement client + shell: bash + run: | + set -euo pipefail + [[ "$MEGA_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C scorpiofs rev-parse HEAD)" = "$SCORPIO_SHA" + test "$(git -C mega2 rev-parse HEAD)" = "$MEGA_SHA" + for target in server client; do + if test "$target" = server; then + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage server-build -- \ + cargo build --manifest-path mega2/Cargo.toml --locked --release --bin mega2 + else + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage client-build -- \ + cargo build --manifest-path scorpiofs/Cargo.toml --locked --release --example mst2_update_measure + fi + done + + - name: Check collector and actual process ownership + shell: bash + run: | + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage fences -- bash -euo pipefail -c ' + python3 -B scorpiofs/tests/mst2-e2e/test_commit_update_bench.py -v + python3 -B scorpiofs/tests/mst2-e2e/test_commit_update_budget.py -v + python3 -B scorpiofs/tests/mst2-e2e/test_commit_update_projection.py -v + MST2_MEASURE_DRIVER="$GITHUB_WORKSPACE/scorpiofs/target/release/examples/mst2_update_measure" \ + cargo test --manifest-path scorpiofs/Cargo.toml --locked --release --example mst2_update_measure + ' + + - name: Measure actual request-bound commit updates on owned deployment + shell: bash + run: | + set -euo pipefail + DRIVER="$GITHUB_WORKSPACE/scorpiofs/target/release/examples/mst2_update_measure" + DRIVER_SHA=$(sha256sum "$DRIVER" | cut -d ' ' -f 1) + python3 -B scorpiofs/tests/mst2-e2e/commit_update_ci.py --execute --projection-traces \ + --run-root "$MST2_OWNED_ROOT" --mega-source "$GITHUB_WORKSPACE/mega2" \ + --mega-sha "$MEGA_SHA" --mega-binary "$GITHUB_WORKSPACE/mega2/target/release/mega2" \ + --driver "$DRIVER" --driver-sha256 "$DRIVER_SHA" --profile "$PROFILE" --rounds "$ROUNDS" \ + --session-started-utc "$MST2_SESSION_STARTED" --session-deadline-utc "$MST2_SESSION_DEADLINE" + + - name: Clean up owned deployment within original H + if: always() + shell: bash + run: | + if test -n "$MST2_OWNED_ROOT" && test -f scorpiofs/tests/mst2-e2e/commit_update_ci.py; then + python3 -B scorpiofs/tests/mst2-e2e/commit_update_ci.py --cleanup --run-root "$MST2_OWNED_ROOT" \ + --session-deadline-utc "$MST2_SESSION_DEADLINE" --rounds "$ROUNDS" + fi diff --git a/.github/workflows/mst2-real-update.yml b/.github/workflows/mst2-real-update.yml new file mode 100644 index 00000000..8b42f5de --- /dev/null +++ b/.github/workflows/mst2-real-update.yml @@ -0,0 +1,154 @@ +name: MST2 real commit update benchmark + +on: + pull_request: + types: [labeled] + workflow_dispatch: + inputs: + mega_sha: + description: Immutable reviewed Mega2 server SHA + required: true + default: 37b61086288b80c10b413472e262a1bf281779eb + profile: + type: choice + description: Fixture size + options: [medium, smoke] + default: medium + rounds: + description: Repetitions (3 to 10) + required: true + default: '3' + session_deadline_utc: + description: Original shared UTC deadline when recovering a failed session + required: false + job_timeout_minutes: + description: Hard job bound; recovery must fit before the original deadline + required: true + default: '240' + +permissions: + contents: read + +jobs: + benchmark: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.head.ref == 'test/mst2-real-commit-update-runner' && + (github.event.label.name == 'run-mst2-real-update-medium' || + startsWith(github.event.label.name, 'resume-mst2-update:'))) + runs-on: ubuntu-latest + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && fromJSON(inputs.job_timeout_minutes) || startsWith(github.event.label.name, 'resume-mst2-update:') && fromJSON(github.event.label.description).timeout_minutes || 240 }} + env: + CARGO_BUILD_JOBS: '2' + CARGO_INCREMENTAL: '0' + CARGO_PROFILE_RELEASE_DEBUG: '0' + NO_PROXY: localhost,127.0.0.1 + MEGA_SHA: ${{ github.event_name == 'pull_request' && '37b61086288b80c10b413472e262a1bf281779eb' || inputs.mega_sha }} + SCORPIO_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + PROFILE: ${{ github.event_name == 'pull_request' && 'medium' || inputs.profile }} + ROUNDS: ${{ github.event_name == 'pull_request' && '3' || inputs.rounds }} + DEADLINE_INPUT: ${{ inputs.session_deadline_utc }} + TIMEOUT_INPUT: ${{ inputs.job_timeout_minutes }} + RESUME_SPEC: ${{ startsWith(github.event.label.name, 'resume-mst2-update:') && github.event.label.description || '' }} + steps: + - name: Set shared session deadline + shell: bash + run: | + python3 - <<'PY' + import json, os, time + from datetime import datetime, timedelta, timezone + now = datetime.now(timezone.utc) + spec = json.loads(os.environ['RESUME_SPEC']) if os.environ['RESUME_SPEC'] else {} + requested = spec.get('session_deadline_utc', os.environ['DEADLINE_INPUT']) + timeout = spec.get('timeout_minutes', int(os.environ['TIMEOUT_INPUT'] or '240')) + if not isinstance(timeout, int) or isinstance(timeout, bool) or not 1 <= timeout <= 240: + raise ValueError('invalid hard job timeout') + deadline = datetime.fromisoformat(requested.replace('Z', '+00:00')) if requested else now + timedelta(minutes=235) + if deadline.tzinfo is None or deadline.utcoffset().total_seconds() != 0: + raise ValueError('shared deadline must be explicit UTC') + remaining = (deadline - now).total_seconds() + if not 0 < remaining <= 235 * 60 or (requested and timeout * 60 > remaining): + raise ValueError('recovery timeout must fit the unextended shared deadline') + if spec and not requested: + raise ValueError('resume label must carry the original shared deadline') + with open(os.environ['GITHUB_ENV'], 'a') as stream: + stream.write('MST2_SESSION_DEADLINE=' + deadline.isoformat() + '\n') + stream.write('MST2_WORK_CLEANUP_DEADLINE_MONOTONIC=' + repr(time.monotonic() + remaining - 15 * 60) + '\n') + stream.write('MST2_OWNED_ROOT=' + os.environ['RUNNER_TEMP'] + '/mst2-real-' + os.environ['GITHUB_RUN_ID'] + '-' + os.environ['GITHUB_RUN_ATTEMPT'] + '\n') + PY + + - uses: actions/checkout@v7 + with: + ref: ${{ env.SCORPIO_SHA }} + path: scorpiofs + persist-credentials: false + + - uses: actions/checkout@v7 + with: + repository: gitmono-dev/mega2 + ref: ${{ env.MEGA_SHA }} + path: mega2 + persist-credentials: false + + - uses: actions/cache@v6 + with: + path: scorpiofs/target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('scorpiofs/Cargo.lock') }} + + - name: Install system dependencies + timeout-minutes: 10 + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev libfuse3-dev protobuf-compiler postgresql-client + + - uses: dtolnay/rust-toolchain@stable + + - name: Build immutable server and measurement client + shell: bash + run: | + set -euo pipefail + [[ "$MEGA_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C scorpiofs rev-parse HEAD)" = "$SCORPIO_SHA" + test "$(git -C mega2 rev-parse HEAD)" = "$MEGA_SHA" + for target in server client; do + if test "$target" = server; then + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage server-build -- \ + cargo build --manifest-path mega2/Cargo.toml --locked --release --bin mega2 + else + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage client-build -- \ + cargo build --manifest-path scorpiofs/Cargo.toml --locked --release --example mst2_update_measure + fi + done + + - name: Check benchmark fences and Git oracle + shell: bash + run: | + python3 -B scorpiofs/tests/mst2-e2e/commit_update_budget.py --rounds "$ROUNDS" --stage fences -- bash -euo pipefail -c ' + python3 -B scorpiofs/tests/mst2-e2e/test_commit_update_bench.py -v + python3 -B scorpiofs/tests/mst2-e2e/test_commit_update_budget.py -v + MST2_MEASURE_DRIVER="$GITHUB_WORKSPACE/scorpiofs/target/release/examples/mst2_update_measure" \ + cargo test --manifest-path scorpiofs/Cargo.toml --locked --release --example mst2_update_measure + ' + + - name: Run real commit update benchmark on owned deployment + shell: bash + run: | + set -euo pipefail + DRIVER="$GITHUB_WORKSPACE/scorpiofs/target/release/examples/mst2_update_measure" + DRIVER_SHA=$(sha256sum "$DRIVER" | cut -d ' ' -f 1) + python3 -B scorpiofs/tests/mst2-e2e/commit_update_ci.py --execute \ + --run-root "$MST2_OWNED_ROOT" --mega-source "$GITHUB_WORKSPACE/mega2" \ + --mega-sha "$MEGA_SHA" --mega-binary "$GITHUB_WORKSPACE/mega2/target/release/mega2" \ + --driver "$DRIVER" --driver-sha256 "$DRIVER_SHA" \ + --profile "$PROFILE" --rounds "$ROUNDS" --session-deadline-utc "$MST2_SESSION_DEADLINE" + + - name: Clean up owned deployment + if: always() + shell: bash + run: | + if test -f scorpiofs/tests/mst2-e2e/commit_update_ci.py; then + python3 -B scorpiofs/tests/mst2-e2e/commit_update_ci.py --cleanup --run-root "$MST2_OWNED_ROOT" \ + --session-deadline-utc "$MST2_SESSION_DEADLINE" --rounds "$ROUNDS" + fi diff --git a/examples/mst2_update_measure.rs b/examples/mst2_update_measure.rs new file mode 100644 index 00000000..93704f4b --- /dev/null +++ b/examples/mst2_update_measure.rs @@ -0,0 +1,486 @@ +//! One measured real-service sync, driven by commit_update_bench.py. +//! The independent expected manifest comes from Git, outside measured work. +//! Local completion checks establish integrity, not offline authorization. + +#![recursion_limit = "256"] + +use std::{collections::BTreeMap, path::PathBuf, sync::Arc, time::Instant}; + +use scorpiofs::snapshot::{ + DurableStore, FetchCoordinator, IncrementalSync, Mst2Client, ScopeCache, SnapshotFile, + SnapshotReader, ViewMeta, +}; + +#[derive(serde::Deserialize, serde::Serialize)] +struct Expected { + files: Vec, + directories: Vec, +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct AuditAuthority { + revision: u8, + domain: String, + scope: String, + snapshot_id: Option, +} + +fn audit_old_view( + expected: &Expected, + root: &std::path::Path, + content: &std::path::Path, +) -> Result<(), Box> { + // Check the explicit shared CAS binding before opening can recover markers. + let view: AuditAuthority = + serde_json::from_slice(&std::fs::read(root.join("authority.json"))?)?; + let cas: AuditAuthority = + serde_json::from_slice(&std::fs::read(content.join("authority.json"))?)?; + let snapshot = view + .snapshot_id + .as_deref() + .ok_or("view binding missing snapshot")?; + if view.revision != 1 + || cas.revision != 1 + || view.domain.len() != 64 + || !view + .domain + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + || view.domain != cas.domain + || view.scope != cas.scope + || cas.snapshot_id.is_some() + || !snapshot.starts_with("sha256:") + || snapshot.len() != 71 + || !snapshot[7..] + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + { + return Err("old view and shared content authority differ".into()); + } + mst2_codec::descriptor::validate_scope(&view.scope)?; + let store = DurableStore::open_with_content(root, content)?; + let closure = store.snapshot_manifest()?; + if !store.is_snapshot_complete()? + || closure.snapshot_id() != snapshot + || closure.descriptor().scope != view.scope + || index(closure.files()) != index(&expected.files) + || !same_directories(closure.directories(), &expected.directories) + { + return Err("old fixed view no longer matches its complete Git oracle".into()); + } + Ok(()) +} + +fn same_directories( + actual: &[scorpiofs::snapshot::SnapshotDirectory], + expected: &[String], +) -> bool { + let mut actual: Vec<_> = actual.iter().map(|d| d.rel_path.as_str()).collect(); + let mut expected: Vec<_> = expected.iter().map(String::as_str).collect(); + actual.sort_unstable(); + expected.sort_unstable(); + actual == expected +} + +fn index(files: &[SnapshotFile]) -> BTreeMap { + files + .iter() + .map(|file| { + ( + file.rel_path.clone(), + (file.fs_kind.clone(), file.size, file.content_digest.clone()), + ) + }) + .collect() +} + +fn elapsed_ms(start: Instant) -> f64 { + start.elapsed().as_secs_f64() * 1000.0 +} + +#[tokio::main] +async fn main() -> std::process::ExitCode { + let mut stage = "arguments"; + match measure(&mut stage).await { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + // Stages are fixed harness labels. Error messages and environment + // values are private; only the actual typed client code is public. + let mut failure = serde_json::json!({ + "record": "measurement_failure", "stage": stage, + }); + if let Some(snapshot) = error.downcast_ref::() { + failure["snapshot_error_code"] = serde_json::json!(format!("{:?}", snapshot.code)); + } + eprintln!("{failure}"); + std::process::ExitCode::FAILURE + } + } +} + +async fn measure(stage: &mut &'static str) -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let mode = args.next().ok_or("sync or audit required")?; + let expected_path = PathBuf::from(args.next().ok_or("expected Git manifest required")?); + let expected: Expected = serde_json::from_slice(&std::fs::read(expected_path)?)?; + if index(&expected.files).len() != expected.files.len() { + return Err("duplicate paths in independent Git manifest".into()); + } + if mode == "audit" { + *stage = "old_complete_view_audit"; + let root = PathBuf::from(args.next().ok_or("old store required")?); + let content = PathBuf::from(args.next().ok_or("old shared content store required")?); + if args.next().is_some() { + return Err( + "audit accepts exactly one expected manifest, view store and content store".into(), + ); + } + audit_old_view(&expected, &root, &content)?; + println!("{}", serde_json::json!({"old_view_integrity": "PASS"})); + return Ok(()); + } + if mode != "sync" || args.next().is_some() { + return Err( + "usage: mst2_update_measure sync EXPECTED | audit EXPECTED STORE CONTENT".into(), + ); + } + + let base = std::env::var("M2_BASE")?; + let scope = std::env::var("M2_SCOPE")?; + let storage = PathBuf::from(std::env::var("M2_STORE_ROOT")?); + let expected_view = std::env::var("M2_EXPECTED_VIEW")?; + let expected_instance = std::env::var("M2_EXPECTED_INSTANCE")?; + let client = Mst2Client::with_token(base, std::env::var("M2_TOKEN").ok()); + *stage = "resolve"; + let started = Instant::now(); + let (reader, resolve_receipt) = match std::env::var("M2_RESOLVE_TRACE_ID") { + Ok(logical_id) => { + let (reader, receipt) = + SnapshotReader::resolve_observed(client, &scope, 3600, &logical_id).await?; + (reader, Some(receipt)) + } + Err(std::env::VarError::NotPresent) => { + (SnapshotReader::resolve(client, &scope, 3600).await?, None) + } + Err(error) => return Err(error.into()), + }; + let resolve_ms = elapsed_ms(started); + if reader.descriptor().namespace_view_id != expected_view + || reader.descriptor().instance_id != expected_instance + || reader.descriptor().scope != scope + { + return Err("stale or wrong resolved snapshot after the publication fence".into()); + } + *stage = "cache_setup"; + let cache_start = Instant::now(); + let context = reader.authorized_context(); + if let Ok(expected_sequence) = std::env::var("M2_EXPECTED_SEQUENCE") { + if context.publication_sequence() != expected_sequence.parse::()? { + return Err("resolved publication sequence differs from the native head fence".into()); + } + } + let scope_dir = context.scope_cache_dir(&storage); + context.bind_scope_cache(&scope_dir)?; + let cache = ScopeCache::open(&scope_dir)?; + let view_dir = context.view_cache_dir(&storage)?; + let store = Arc::new(DurableStore::open_for_reader( + &view_dir, + scope_dir.join("blobs"), + &reader, + )?); + let cache_setup_ms = elapsed_ms(cache_start); + *stage = "metadata"; + let metadata_start = Instant::now(); + let before_metadata_bytes = reader.client().received_bytes(); + let mut sync = IncrementalSync::new(&reader, &cache); + let closure = sync.sync_snapshot().await?; + let metadata_ms = elapsed_ms(metadata_start); + let metadata_ready_ms = elapsed_ms(started); + let metadata_bytes = reader.client().received_bytes() - before_metadata_bytes; + let acquisition = sync.meters(); + let proof = sync.closure_meters(); + // Keep the independent oracle outside each reported timing segment, but + // account for it explicitly in overall wall time. + *stage = "metadata_oracle"; + let oracle_start = Instant::now(); + if index(closure.files()) != index(&expected.files) + || closure.files().len() != expected.files.len() + || !same_directories(closure.directories(), &expected.directories) + { + return Err("incremental closure differs from the independent fixed Git tree".into()); + } + let metadata_oracle_ms = elapsed_ms(oracle_start); + *stage = "hydrate"; + let content_start = Instant::now(); + let before_content_bytes = reader.client().received_bytes(); + let frames = + reader.capabilities().features.objects && reader.capabilities().features.chunk_reads; + let client = reader.client().clone(); + let encoding = reader.encoding_hint().map(str::to_string); + let sid = reader.snapshot_id().to_owned(); + let report = if frames { + store + .hydrate_snapshot_batches( + &reader, + &closure, + 4, + 4, + move |batch| { + let client = client.clone(); + let sid = sid.clone(); + let encoding = encoding.clone(); + Box::pin(async move { + let items: Vec<_> = batch + .iter() + .map(|f| (format!("/{}", f.rel_path), f.content_digest.clone())) + .collect(); + let got = client.objects(&sid, &items, encoding.as_deref()).await?; + let mut out = std::collections::HashMap::new(); + for f in &batch { + let want = + scorpiofs::snapshot::frames::parse_digest(&f.content_digest)?; + let data = got.get(&want).ok_or_else(|| { + scorpiofs::snapshot::SnapshotError::new( + scorpiofs::snapshot::SnapshotErrorCode::DigestMismatch, + "object batch omitted an expected Git content digest", + ) + })?; + out.insert(f.content_digest.clone(), Arc::new(data.clone())); + } + Ok(out) + }) + }, + { + let reader = reader.clone(); + move |file| { + let reader = reader.clone(); + Box::pin(async move { + Ok(Arc::new( + reader + .read_file_frames( + &file.rel_path, + &file.content_digest, + file.size, + ) + .await?, + )) + }) + } + }, + ) + .await? + } else { + let coordinator = FetchCoordinator::new(reader.clone(), 4); + store + .hydrate_snapshot_concurrent(&reader, &closure, 4, move |file| { + let coordinator = coordinator.clone(); + Box::pin(async move { coordinator.fetch(file, false).await }) + }) + .await? + }; + let hydration_ms = elapsed_ms(content_start); + let content_bytes = reader.client().received_bytes() - before_content_bytes; + let view = ViewMeta { + snapshot_id: reader.snapshot_id().to_owned(), + namespace_view_id: reader.descriptor().namespace_view_id.clone(), + scope: scope.clone(), + lease_id: reader.lease_id().to_owned(), + }; + *stage = "completion_audit"; + let audit_start = Instant::now(); + store.pin(&view)?; + if !store.is_snapshot_complete()? { + return Err("durable descriptor/page/content/pin completion audit failed".into()); + } + let completion_audit_ms = elapsed_ms(audit_start); + let durable_complete_ms = elapsed_ms(started); + println!( + "{}", + serde_json::json!({ + "record": "mst2_real_update", "snapshot_id": reader.snapshot_id(), + "driver_source_digest": scorpiofs::snapshot::durable::digest_of(include_bytes!("mst2_update_measure.rs")), + "namespace_view_id": reader.descriptor().namespace_view_id, + "resolve_trace_receipt": resolve_receipt, + "descriptor_bytes_hex": hex::encode(closure.descriptor_bytes()), + "publication_sequence": context.publication_sequence(), + "store": view_dir, "content_store": store.content_dir(), "resolve_ms": resolve_ms, + "cache_setup_ms": cache_setup_ms, "metadata_ms": metadata_ms, + "metadata_ready_ms": metadata_ready_ms, "metadata_bytes": metadata_bytes, + "metadata_oracle_ms": metadata_oracle_ms, + "hydration_ms": hydration_ms, "completion_audit_ms": completion_audit_ms, + "durable_complete_ms": durable_complete_ms, + "durable_complete_scope": "resolve through full hydration and completion audit, including the separately reported Git manifest comparison", + "metadata_acquisition": {"traversal_nodes": acquisition.traversal_nodes, + "fetched_pages": acquisition.fetched_pages, "reused_pages": acquisition.reused_pages, + "reused_subtrees": acquisition.reused_subtrees, + "closure_index_reads": acquisition.closure_index_reads, + "closure_index_read_bytes": acquisition.closure_index_read_bytes, + "closure_index_writes": acquisition.closure_index_writes, + "closure_index_write_bytes": acquisition.closure_index_write_bytes, + "pin_set_reads": acquisition.pin_set_reads, + "page_rehashes": acquisition.page_rehashes, + "unique_page_rehashes": acquisition.unique_page_rehashes, + "page_rehash_bytes": acquisition.page_rehash_bytes}, + "full_closure_proof": proof, "content_bytes": content_bytes, + "content_delivery": if frames {"object-batches-and-chunk-frames"} else {"raw-concurrent"}, + "fetched_content_units": report.fetched, "resumed_files": report.resumed, + "logical_files": report.total_files, "logical_bytes": report.bytes_total, + "full_snapshot_complete": true, "git_manifest_equal": true, + "fuse_mount": "NOT_RUN", "offline_authorization": false + }) + ); + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use std::{path::Path, process::Command}; + + use mst2_codec::{ + descriptor::ServingDescriptor, + metapage::{page_id, Entry, EntryKind, Page}, + }; + use scorpiofs::snapshot::{durable::digest_of, ValidatedSnapshotClosure}; + + use super::*; + + async fn fixture(root: &Path, content: &Path, version: u8, body: &[u8]) -> Expected { + let digest = digest_of(body); + let raw = Page::Leaf { + entries: vec![Entry::file( + EntryKind::Regular, + b"old.txt", + body.len() as u64, + scorpiofs::snapshot::frames::parse_digest(&digest).unwrap(), + )], + } + .encode() + .unwrap(); + let pid = page_id(&raw); + let descriptor = ServingDescriptor { + instance_uuid: *uuid::Uuid::parse_str("11111111-2222-4333-8444-555555555555") + .unwrap() + .as_bytes(), + namespace_view_id: [version; 32], + scope: "/project".into(), + metadata_root: pid, + }; + let pages = BTreeMap::from([(format!("sha256:{}", hex::encode(pid)), raw)]); + let closure = + ValidatedSnapshotClosure::from_canonical_pages(&descriptor.encode().unwrap(), pages) + .unwrap(); + let store = DurableStore::open_with_content(root, content).unwrap(); + let view = ViewMeta { + snapshot_id: closure.snapshot_id().into(), + namespace_view_id: closure.descriptor().namespace_view_id.clone(), + scope: "/project".into(), + lease_id: "local-integrity-fixture".into(), + }; + store + .hydrate_snapshot_with(&view, &closure, |_| std::future::ready(Ok(body.to_vec()))) + .await + .unwrap(); + assert!(store.is_snapshot_complete().unwrap()); + for (dir, snapshot) in [(root, Some(closure.snapshot_id())), (content, None)] { + std::fs::write( + dir.join("authority.json"), + serde_json::to_vec(&serde_json::json!({ + "revision": 1, "domain": "a".repeat(64), + "scope": "/project", "snapshot_id": snapshot, + })) + .unwrap(), + ) + .unwrap(); + } + Expected { + files: vec![SnapshotFile { + rel_path: "old.txt".into(), + fs_kind: "regular".into(), + size: body.len() as u64, + content_digest: digest, + }], + directories: vec![String::new()], + } + } + + #[tokio::test] + async fn real_audit_process_preserves_old_shared_cas_and_rejects_wrong_or_corrupt_content() { + let driver = std::env::var_os("MST2_MEASURE_DRIVER") + .map(PathBuf::from) + .unwrap_or_else(|| { + std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .join(format!( + "mst2_update_measure{}", + std::env::consts::EXE_SUFFIX + )) + }); + assert!( + driver.is_file(), + "build the driver or set MST2_MEASURE_DRIVER before example tests" + ); + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("old-view"); + let content = temp.path().join("scope-blobs"); + let original = b"original fixed snapshot bytes"; + let expected = fixture(&root, &content, 0x22, original).await; + fixture( + &temp.path().join("new-view"), + &content, + 0x23, + b"new commit bytes", + ) + .await; + let manifest = temp.path().join("old-expected.json"); + std::fs::write(&manifest, serde_json::to_vec(&expected).unwrap()).unwrap(); + let audit = |cas: &Path| { + Command::new(&driver) + .args(["audit"]) + .arg(&manifest) + .arg(&root) + .arg(cas) + .output() + .unwrap() + }; + let good = audit(&content); + assert!(good.status.success()); + assert_eq!( + serde_json::from_slice::(&good.stdout).unwrap(), + serde_json::json!({"old_view_integrity": "PASS"}) + ); + let wrong = temp.path().join("other-domain"); + std::fs::create_dir(&wrong).unwrap(); + std::fs::write( + wrong.join("authority.json"), + serde_json::to_vec(&serde_json::json!({ + "revision": 1, "domain": "b".repeat(64), + "scope": "/project", "snapshot_id": null, + })) + .unwrap(), + ) + .unwrap(); + assert!(!audit(&wrong).status.success()); + assert!( + audit(&content).status.success(), + "wrong CAS must not recover the real view" + ); + std::fs::write( + content.join( + expected.files[0] + .content_digest + .strip_prefix("sha256:") + .unwrap(), + ), + vec![0u8; original.len()], + ) + .unwrap(); + let corrupt = audit(&content); + assert!(!corrupt.status.success()); + let failure: serde_json::Value = serde_json::from_slice(&corrupt.stderr).unwrap(); + assert_eq!(failure["record"], "measurement_failure"); + assert_eq!(failure["stage"], "old_complete_view_audit"); + } +} diff --git a/tests/mst2-e2e/commit_update_bench.py b/tests/mst2-e2e/commit_update_bench.py new file mode 100644 index 00000000..d3c890f0 --- /dev/null +++ b/tests/mst2-e2e/commit_update_bench.py @@ -0,0 +1,780 @@ +#!/usr/bin/env python3 +"""Measure real commit updates against an explicitly isolated, running Mega2. + +Default is plan-only. Execute never starts/stops services, creates cloud or DB +resources, changes server configuration, force-pushes, or deletes old results. +Build examples/mst2_update_measure.rs first. Tokens and PostgreSQL credentials +are inherited through M2_TOKEN/M2_GIT_TOKEN and PG* environment variables. +""" + +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import hashlib +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +import time +from urllib.parse import urlsplit +import uuid + +import commit_update_budget as budget_module +import commit_update_projection as projection_module + +try: + import tomllib +except ModuleNotFoundError: + from pip._vendor import tomli as tomllib + + +IDENTITY_SQL = """BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path = public; +SELECT json_agg(row_to_json(x)) FROM ( + SELECT r.path, r.ref_commit_hash AS commit, r.ref_tree_hash AS tree, + current_database() AS database, c.tree AS commit_tree, + CASE WHEN r.path='/' THEN encode(t.sub_trees,'hex') ELSE NULL END AS raw_tree + FROM mega_refs r LEFT JOIN mega_tree t ON t.tree_id=r.ref_tree_hash + LEFT JOIN mega_commit c ON c.commit_id=r.ref_commit_hash + WHERE r.ref_name='refs/heads/main' AND r.path IN ('/', '/project') AND NOT r.is_cl + ORDER BY r.path +) x; +COMMIT;""" + +NATIVE_SQL = """BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY; +SET LOCAL search_path = public; +SELECT COALESCE((SELECT row_to_json(x) FROM ( + SELECT h.instance_id, h.sequence, h.writer_epoch, h.root_commit, h.root_tree, + h.state, h.certificate_receipt_id, n.root_commit AS certificate_commit, + n.root_tree AS certificate_tree, n.sequence AS certificate_sequence, + n.instance_id AS certificate_instance, n.path_commit, n.path_tree, + n.origin_path, n.origin_ref, p.sequence AS origin_sequence, + n.receipt_id AS certificate_id, n.namespace AS certificate_namespace, + n.writer_epoch AS certificate_epoch, n.old_root_commit, n.old_path_commit, + p.id AS receipt_id, p.namespace AS receipt_namespace, p.old_oid, p.new_oid, + p.writer_epoch AS receipt_epoch, p.writer_kind, p.request_digest, + p.request_digest_version, + p.native_certificate_version, o.id AS outbox_id, + o.namespace AS outbox_namespace, o.sequence AS outbox_sequence + FROM mst2_native_head h + LEFT JOIN mst2_native_publication n ON n.receipt_id=h.certificate_receipt_id + LEFT JOIN mst2_publication p ON p.id=n.receipt_id + LEFT JOIN mst2_publication_outbox o ON o.operation_id=p.operation_id + WHERE h.namespace='/' +) x), 'null'::json); +COMMIT;""" + + +SNAPSHOT_ERROR_CODES = frozenset({ + "ScopeInvalid", "InvalidRequest", "LimitExceeded", "Unauthenticated", "ScopeForbidden", + "ViewNotFound", "SnapshotNotReady", "SnapshotGone", "PathNotFound", "NotDirectory", + "UnsupportedEntry", "LeaseUnknown", "LeaseExpired", "CursorInvalid", "CursorStale", + "ProofBudgetExceeded", "DigestMismatch", "IntegrityError", "ObjectUnavailable", + "RangeNotSupported", "SymlinkTraversal", "DurableViewConflict", "TemporaryUnavailable", "Internal", +}) +MEASUREMENT_STAGES = frozenset({ + "arguments", "resolve", "cache_setup", "metadata", "metadata_oracle", "hydrate", + "completion_audit", "old_complete_view_audit", +}) + + +class CommandFailure(RuntimeError): + """Closed diagnostic fields; never command arguments or child messages.""" + + def __init__(self, program, status, stderr): + self.details = {"command": program, "exit_status": status} + if program == "mst2_update_measure" and len(stderr) <= 4096: + try: + failure = json.loads(stderr) + except (json.JSONDecodeError, UnicodeDecodeError): + failure = None + if (isinstance(failure, dict) + and set(failure) in ({"record", "stage"}, {"record", "stage", "snapshot_error_code"}) + and failure["record"] == "measurement_failure" + and isinstance(failure["stage"], str) + and failure["stage"] in MEASUREMENT_STAGES + and ("snapshot_error_code" not in failure + or (isinstance(failure["snapshot_error_code"], str) + and failure["snapshot_error_code"] in SNAPSHOT_ERROR_CODES))): + self.details["measurement_stage"] = failure["stage"] + if "snapshot_error_code" in failure: + self.details["snapshot_error_code"] = failure["snapshot_error_code"] + super().__init__(program + " failed") + + +class PhaseFailure(AssertionError): + """A fixed harness phase, without child output or exception text.""" + + def __init__(self, phase, error): + self.phase = phase + self.failure_type = type(error).__name__ + self.details = error.details if isinstance(error, CommandFailure) else {} + super().__init__(phase + " failed") + + +@contextmanager +def phase(name): + try: + yield + except PhaseFailure: + raise + except Exception as error: + raise PhaseFailure(name, error) from None + + +def failure_record(error): + record = {"execution_failed": True, "error_type": type(error).__name__} + if isinstance(error, PhaseFailure): + record.update(error_type=error.failure_type, phase=error.phase) + record.update(error.details) + elif isinstance(error, CommandFailure): + record.update(error.details) + return record + + +def clean_env(extra=None): + env = {k: os.environ[k] for k in ("HOME", "USER", "LOGNAME", "PATH", "LANG", "LC_ALL", "TZ") + if k in os.environ} + env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL="/dev/null", + GIT_TERMINAL_PROMPT="0", NO_PROXY="127.0.0.1,localhost", + no_proxy="127.0.0.1,localhost") + if extra: + env.update(extra) + return env + + +def command(args, deadline, env=None, data=None): + # Each child owns a process group; a stalled Git/HTTP child cannot survive + # the shared wall-clock deadline. Captured errors never echo bearer headers. + status, out, error_output = budget_module.run_process( + args, min(deadline, time.monotonic() + 1800), env=env or clean_env(), data=data) + if status: + name = Path(str(args[0])).name.removesuffix(".exe") + program = name if name in {"git", "psql", "docker", "mst2_update_measure"} else "external_command" + raise CommandFailure(program, status, error_output) + return out + + +def git(repo, deadline, *args, env=None, data=None): + return command(["git", "-C", str(repo), *args], deadline, env=env, data=data) + + +def fsync_tree(root, deadline): + """Flush regular files and directories, without following symlinks.""" + files = directories = size = 0 + if root.is_symlink() or not root.is_dir(): + raise AssertionError("flush root must be a real private directory") + def fail(error): + raise error + for base, _, names in os.walk(root, topdown=False, followlinks=False, onerror=fail): + for name in names: + if time.monotonic() >= deadline: + raise TimeoutError("Git durable worktree flush exceeded the shared deadline") + path = Path(base) / name + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + continue # Its inode/entry is committed by the parent dir fsync. + if not stat.S_ISREG(mode): + raise AssertionError("unexpected special node in private Git baseline") + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + size += os.fstat(fd).st_size + os.fsync(fd) + finally: + os.close(fd) + files += 1 + fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(fd) + finally: + os.close(fd) + directories += 1 + return {"regular_files": files, "directories": directories, "file_bytes": size} + + +def endpoint_pair(base_url, git_url): + base, remote = urlsplit(base_url), urlsplit(git_url) + for endpoint in (base, remote): + if (endpoint.scheme != "http" or endpoint.hostname != "127.0.0.1" or not endpoint.port + or endpoint.username or endpoint.password or endpoint.query or endpoint.fragment): + raise ValueError("only explicit credential-free loopback HTTP endpoints are accepted") + if base.netloc != remote.netloc or base.path not in ("", "/") or remote.path != "/project": + raise ValueError("Git and MST/2 must address the same isolated service /project") + return base.port + + +def service_binding(options): + """Bind HTTP socket, process start identity, effective DB and MST instance.""" + proc = Path(f"/proc/{options.service_pid}") + started = proc.joinpath("stat").read_text().rsplit(") ", 1)[1].split()[19] + argv = [x.decode() for x in proc.joinpath("cmdline").read_bytes().split(b"\0") if x] + if "--config" not in argv or any(x.startswith("--profile") for x in argv): + raise AssertionError("service must load one explicit literal config, without profile merging") + config_path = Path(argv[argv.index("--config") + 1]).resolve(strict=True) + raw = config_path.read_bytes() + config = tomllib.loads(raw.decode()) + inherited = dict(x.decode().split("=", 1) for x in proc.joinpath("environ").read_bytes().split(b"\0") if x) + allowed = {"MEGA_DATABASE__DB_URL", "MEGA_REDIS__URL", "MEGA_BASE_DIR", "MEGA_CACHE_DIR", + "MEGA_GIT_OBJECT_CACHE_PREFIX"} + if "MEGA_PROFILE" in inherited or any(k.startswith("MEGA_") and k not in allowed for k in inherited): + raise AssertionError("unreviewed service environment overrides literal config") + db = urlsplit(inherited.get("MEGA_DATABASE__DB_URL", config["database"]["db_url"])) + if (db.scheme not in ("postgres", "postgresql") or db.query or db.fragment + or db.hostname != os.environ.get("PGHOST") or db.port != int(os.environ["PGPORT"]) + or db.username != os.environ.get("PGUSER") + or db.path != "/" + options.database or os.environ.get("PGDATABASE") != options.database): + raise AssertionError("read-only identity DB does not match the running service") + if (not config["mst2"]["enabled"] or config["mst2"]["instance_uuid"] != options.instance_id + or bool(config["mst2"]["publication_enabled"]) != (options.publication_mode == "native")): + raise AssertionError("service MST/2 instance/publication mode differs from explicit arguments") + if config["monorepo"]["object_format"] != "sha1" or config["monorepo"]["push_policy"] != "trunk": + raise AssertionError("benchmark requires SHA-1 trunk ingest") + projection = getattr(options, "projection_traces", False) + if projection and (config["mst2"].get("projection_observation_enabled") is not True + or not inherited.get("MEGA_CACHE_DIR")): + raise AssertionError("typed projection sink must use the owned service's explicit cache") + sockets = set() + for fd in proc.joinpath("fd").iterdir(): + try: + target = fd.readlink().as_posix() + except FileNotFoundError: + continue + if target.startswith("socket:["): + sockets.add(target) + port = endpoint_pair(options.base_url, options.git_url) + listener = False + for line in Path("/proc/net/tcp").read_text().splitlines()[1:]: + fields = line.split() + address, number = fields[1].split(":") + if address == "0100007F" and int(number, 16) == port and fields[3] == "0A": + listener = f"socket:[{fields[9]}]" in sockets + if not listener: + raise AssertionError("requested loopback listener is not owned by the explicit service PID") + binding = {"pid": options.service_pid, "starttime_ticks": started, + "exe": str(proc.joinpath("exe").resolve()), + "config_sha256": hashlib.sha256(raw).hexdigest()} + if projection: + binding["projection_cache"] = inherited["MEGA_CACHE_DIR"] + return binding + + +def query(sql, deadline, env=None): + if env is None: + env = clean_env({k: v for k, v in os.environ.items() if k.startswith("PG")}) + raw = command(["psql", "-X", "-q", "-A", "-t", "-v", "ON_ERROR_STOP=1"], + deadline, env=env, data=sql.encode()) + try: + return json.loads(raw) + except json.JSONDecodeError: + # Never echo SQL results, connection strings or credentials in CI. + phase = "native publication" if sql == NATIVE_SQL else "Git identity" + raise AssertionError(f"{phase} fence returned invalid JSON") from None + + +def validate_identity(rows, commit, tree, database): + if len(rows) != 2 or {row["path"] for row in rows} != {"/", "/project"}: + raise AssertionError("exact global and project main refs are required") + rows = {row["path"]: row for row in rows} + if any(row["database"] != database or row["commit_tree"] != row["tree"] for row in rows.values()): + raise AssertionError("database or stored commit/tree differs") + if rows["/project"]["commit"] != commit or rows["/project"]["tree"] != tree: + raise AssertionError("actual project main is stale or differs from the pushed Git commit") + raw = bytes.fromhex(rows["/"]["raw_tree"]) + if hashlib.sha1(b"tree " + str(len(raw)).encode() + b"\0" + raw).hexdigest() != rows["/"]["tree"]: + raise AssertionError("global raw tree does not match its Git object ID") + project = [] + cursor = 0 + while cursor < len(raw): + separator = raw.index(b" ", cursor) + end_name = raw.index(b"\0", separator + 1) + if end_name + 21 > len(raw): + raise AssertionError("truncated global tree") + if raw[separator + 1:end_name] == b"project": + project.append((raw[cursor:separator], raw[end_name + 1:end_name + 21].hex())) + cursor = end_name + 21 + if project != [(b"40000", tree)]: + raise AssertionError("global tree /project is not the fixed local Git tree") + root = rows["/"] + return {"project_commit": commit, "project_tree": tree, "global_commit": root["commit"], + "global_tree": root["tree"], "namespace_view_id": "sha256:" + + hashlib.sha256(b"mega.mst2.namespaceview\0" + root["commit"].encode()).hexdigest()} + + +def validate_native(native, identity, instance_id, after_push, previous=None): + if (not native or native["instance_id"] != instance_id + or native["root_commit"] != identity["global_commit"] + or native["root_tree"] != identity["global_tree"] + or native["writer_epoch"] != 1 or not isinstance(native["sequence"], int) + or native["sequence"] < 0): + raise AssertionError("native head does not bind the same global commit/tree/instance") + if not after_push and native["state"] == "INITIALIZING" and native["certificate_receipt_id"] is None: + return + if (native["state"] != "READY" or not native["certificate_receipt_id"] or not native["outbox_id"] + or native["certificate_commit"] != identity["global_commit"] + or native["certificate_tree"] != identity["global_tree"] + or native["certificate_instance"] != instance_id + or native["certificate_sequence"] != native["sequence"] + or native["path_commit"] != identity["project_commit"] + or native["path_tree"] != identity["project_tree"] + or native["origin_path"] != "/project" or native["origin_ref"] != "refs/heads/main" + or native["native_certificate_version"] != 1 + or native["certificate_id"] != native["certificate_receipt_id"] + or native["receipt_id"] != native["certificate_receipt_id"] + or native["certificate_namespace"] != "/" + or native["certificate_epoch"] != native["writer_epoch"] + or native["receipt_epoch"] != native["writer_epoch"] + or native["writer_kind"] != "trunk_push" + or native["request_digest_version"] != 1 + or not re.fullmatch(r"sha256:[0-9a-f]{64}", native["request_digest"] or "") + or native["old_oid"] is None or native["old_oid"] != native["old_root_commit"] + or native["new_oid"] is None or native["new_oid"] != native["path_commit"] + or native["old_path_commit"] == native["path_commit"] + or native["receipt_namespace"] != native["origin_path"] + or native["outbox_namespace"] != "/project" + or native["outbox_sequence"] != native["origin_sequence"]): + raise AssertionError("native publication certificate/receipt/outbox is incomplete or mismatched") + if previous and (native["old_root_commit"] != previous["global_commit"] + or native["old_path_commit"] != previous["project_commit"]): + raise AssertionError("native publication does not continue the previously observed fixed root/path") + + +def expected_manifest(repo, commit, deadline): + records = git(repo, deadline, "ls-tree", "-rz", "-t", commit).split(b"\0") + parsed = [] + directories = [""] + unique = {} + for record in filter(None, records): + meta, path = record.split(b"\t", 1) + mode, kind, oid = meta.split() + if kind == b"tree": + directories.append(path.decode()) + continue + if kind != b"blob" or mode not in (b"100644", b"100755", b"120000"): + raise AssertionError("unexpected Git entry kind") + parsed.append((path.decode(), mode, oid)) + unique[oid] = None + # A single Git cat-file invocation avoids one process per file and is + # independent of MST/2 metadata/content digest implementations. + raw = git(repo, deadline, "cat-file", "--batch", data=b"\n".join(unique) + b"\n") + cursor = 0 + for oid in unique: + end = raw.index(b"\n", cursor) + got, kind, count = raw[cursor:end].split() + size = int(count) + body = raw[end + 1:end + 1 + size] + if got != oid or kind != b"blob" or len(body) != size or raw[end + 1 + size:end + 2 + size] != b"\n": + raise AssertionError("truncated or wrong independent Git blob") + unique[oid] = (size, "sha256:" + hashlib.sha256(body).hexdigest()) + cursor = end + 2 + size + return {"files": [{"rel_path": path, "fs_kind": "symlink" if mode == b"120000" else + "executable" if mode == b"100755" else "regular", "size": unique[oid][0], + "content_digest": unique[oid][1]} for path, mode, oid in parsed], + "directories": directories} + + +def verify_worktree(worktree, expected, deadline=None): + wanted = {f["rel_path"]: f for f in expected["files"]} + found = set() + def fail(error): + raise error + for base, directories, names in os.walk(worktree, followlinks=False, onerror=fail): + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("Git byte oracle exceeded its operation budget") + if Path(base) == worktree: + directories[:] = [d for d in directories if d != ".git"] + names = [n for n in names if n != ".git"] + links = [d for d in directories if (Path(base) / d).is_symlink()] + directories[:] = [d for d in directories if d not in links] + names.extend(links) + for name in names: + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("Git byte oracle exceeded its operation budget") + path = Path(base) / name + rel = path.relative_to(worktree).as_posix() + if rel not in wanted: + raise AssertionError("unexpected Git worktree entry") + file = wanted[rel] + mode = path.lstat().st_mode + if file["fs_kind"] == "symlink": + if not stat.S_ISLNK(mode): + raise AssertionError("Git symlink mode differs from the fixed commit") + data = os.fsencode(os.readlink(path)) + else: + if not stat.S_ISREG(mode) or bool(mode & 0o111) != (file["fs_kind"] == "executable"): + raise AssertionError("Git regular/executable mode differs from the fixed commit") + data = path.read_bytes() + if len(data) != file["size"] or "sha256:" + hashlib.sha256(data).hexdigest() != file["content_digest"]: + raise AssertionError("Git worktree content differs from the fixed commit") + found.add(rel) + if found != set(wanted): + raise AssertionError("Git worktree omitted expected content") + + +def create_version(repo, round_number, version, smoke, deadline): + prefix = f"r{round_number:02}" + user = clean_env({"GIT_AUTHOR_NAME": "MST2 benchmark", "GIT_COMMITTER_NAME": "MST2 benchmark", + "GIT_AUTHOR_EMAIL": "benchmark@example.invalid", "GIT_COMMITTER_EMAIL": "benchmark@example.invalid"}) + if version == "v1": + git(repo, deadline, "read-tree", "--empty") + modules, buckets, files, size = (8, 1, 8, 1024) if smoke else (64, 8, 32, 16384) + for module in range(modules): + for bucket in range(buckets): + directory = repo / prefix / f"m{module:03}" / f"d{bucket:02}" + directory.mkdir(parents=True) + for number in range(files): + if time.monotonic() >= deadline: + raise TimeoutError("fixture generation exceeded the shared deadline") + name = f"{prefix}/m{module:03}/d{bucket:02}/f{number:03}" + block = hashlib.sha256(name.encode()).digest() + (repo / name).write_bytes(block * (size // len(block))) + wide = repo / prefix / "wide" + wide.mkdir() + for number in range(129): + (wide / f"f{number:03}").write_bytes(hashlib.sha256(f"{prefix}/wide/{number}".encode()).digest()) + (repo / prefix / "large.bin").write_bytes(hashlib.sha256(prefix.encode()).digest() * + ((65536 if smoke else 2097152) // 32)) + git(repo, deadline, "add", "--", prefix) + elif version == "v2": + path = repo / prefix / "m000/d00/f000" + body = bytearray(path.read_bytes()) + body[0] ^= 1 + path.write_bytes(body) + git(repo, deadline, "add", "--", f"{prefix}/m000/d00/f000") + else: + git(repo, deadline, "mv", "--", f"{prefix}/m001", f"{prefix}/renamed-m001") + parent = git(repo, deadline, "rev-parse", "HEAD").decode().strip() + tree = git(repo, deadline, "write-tree").decode().strip() + # Preserve raw empty directories and a logical directory alias in every + # Git commit. Git checkout omits empty trees; MST/2 full closure must not. + empty = git(repo, deadline, "mktree", data=b"").decode().strip() + alias = git(repo, deadline, "rev-parse", f"{tree}:{prefix}/m007").decode().strip() + entries = [r for r in git(repo, deadline, "ls-tree", "-z", tree).split(b"\0") if r] + entries += [f"040000 tree {empty}\tempty-a".encode(), f"040000 tree {empty}\tempty-b".encode(), + f"040000 tree {alias}\talias-{prefix}-m007".encode()] + tree = git(repo, deadline, "mktree", "-z", data=b"\0".join(entries) + b"\0").decode().strip() + commit = git(repo, deadline, "commit-tree", tree, "-p", parent, env=user, + data=f"MST2 benchmark round {round_number} {version}\n".encode()).decode().strip() + git(repo, deadline, "update-ref", "refs/heads/main", commit, parent) + return commit, tree + + +def percentile(values, percentage): + return sorted(values)[math.ceil(len(values) * percentage / 100) - 1] + + +def durable_verified(operation, oracle): + """The common end point includes each side's immediate full-byte oracle.""" + started = time.monotonic() + measured = operation() + oracle_started = time.monotonic() + oracle(measured) + measured["durable_byte_oracle_ms"] = (time.monotonic() - oracle_started) * 1000 + measured["durable_verified_ms"] = (time.monotonic() - started) * 1000 + return measured + + +def driver_binding(options): + if not re.fullmatch(r"[0-9a-f]{64}", options.driver_sha256): + raise ValueError("a fixed measurement binary SHA-256 is required") + if options.driver.is_symlink() or not options.driver.is_file(): + raise ValueError("measurement binary must be a real file") + if hashlib.sha256(options.driver.read_bytes()).hexdigest() != options.driver_sha256: + raise AssertionError("measurement binary differs from the reviewed fixed artifact") + + +def execute(options): + if sys.platform != "linux" or not options.isolated_deployment: + raise ValueError("--execute requires Linux and explicit --isolated-deployment") + if (options.database in ("mega2", "mega2_test", "mono", "postgres") + or not re.fullmatch(r"[a-z][a-z0-9_]*", options.database)): + raise ValueError("an explicitly isolated benchmark database is required") + if uuid.UUID(options.instance_id).int == 0 or str(uuid.UUID(options.instance_id)) != options.instance_id: + raise ValueError("instance UUID must be canonical and nonnil") + if not re.fullmatch(r"[0-9a-f]{40}", options.expect_initial_commit): + raise ValueError("expected initial project commit must be a fixed SHA-1") + endpoint_pair(options.base_url, options.git_url) + projection = getattr(options, "projection_traces", False) + if projection and (options.publication_mode != "native" + or not callable(getattr(options, "finalize_projection", None))): + raise ValueError("projection collection requires the owned native runner and final drain") + budget = budget_module.from_options(options) + budget.require(options.rounds * budget_module.ROUND_SECONDS + + budget_module.REPORT_RESERVE + budget_module.CLEANUP_RESERVE + + budget_module.MARGIN) + started = time.monotonic() + # This one deadline includes all preflight, fixture, publish, client, Git, + # audit and repeated-round work. Never grant a fresh four hours per round. + measurement_limit = min(budget.measurement_deadline, started + options.deadline_seconds) + deadline = measurement_limit + owner = service_binding(options) + collector = projection_module.ProjectionCollector(owner["projection_cache"]) if projection else None + run_id = str(uuid.uuid4()) if projection else None + driver_binding(options) + root = options.run_root.parent.resolve(strict=True) / options.run_root.name + checkout = Path(__file__).resolve().parents[2] + if root.exists() or root.is_relative_to(checkout) or not root.parent.is_dir(): + raise ValueError("use a nonexistent private run root outside the source checkout") + token = os.environ.get("M2_GIT_TOKEN", os.environ.get("M2_TOKEN", "")) + if not token or not os.environ.get("M2_TOKEN"): + raise ValueError("private M2_GIT_TOKEN/M2_TOKEN authentication is required") + git_env = clean_env({"GIT_CONFIG_COUNT": "3", "GIT_CONFIG_KEY_0": "http.extraHeader", + "GIT_CONFIG_VALUE_0": "Authorization: Bearer " + token, + "GIT_CONFIG_KEY_1": "http.followRedirects", "GIT_CONFIG_VALUE_1": "false", + "GIT_CONFIG_KEY_2": "credential.helper", "GIT_CONFIG_VALUE_2": ""}) + def tip(): + output = command(["git", "ls-remote", options.git_url, "refs/heads/main"], deadline, env=git_env) + lines = output.decode().splitlines() + if len(lines) != 1 or lines[0].split()[1] != "refs/heads/main": + raise AssertionError("expected exactly one project main ref") + return lines[0].split()[0] + if tip() != options.expect_initial_commit: + raise AssertionError("isolated service target moved before any workload mutation") + with phase("initial_complete_identity"): + initial = query(IDENTITY_SQL, deadline) + project = next(row for row in initial if row["path"] == "/project") + identity = validate_identity(initial, options.expect_initial_commit, project["tree"], options.database) + if options.publication_mode == "native": + validate_native(query(NATIVE_SQL, deadline), identity, options.instance_id, False) + root.mkdir(mode=0o700) + (root / ".mst2-real-update-owned").write_text(root.name + "\n") + fixture = root / "fixture" + command(["git", "clone", "--no-checkout", "--single-branch", "--branch", "main", + options.git_url, str(fixture)], deadline, env=git_env) + if git(fixture, deadline, "rev-parse", "HEAD").decode().strip() != options.expect_initial_commit: + raise AssertionError("fixture clone differs from the inspected target") + records = [] + current = options.expect_initial_commit + output = root / "measurements.jsonl" + def emit(record): + with output.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record, sort_keys=True) + "\n") + print(json.dumps(record, sort_keys=True), flush=True) + emit({"record": "environment", "profile": options.profile, "rounds": options.rounds, + "publication_mode": options.publication_mode, "service_binding": owner, + "instrumentation_mode": "typed-projection-writer-v1" if projection else "disabled", + "driver_sha256": hashlib.sha256(options.driver.read_bytes()).hexdigest(), + "driver_source_sha256": hashlib.sha256((checkout / "examples/mst2_update_measure.rs").read_bytes()).hexdigest(), + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "git_version": command(["git", "--version"], deadline).decode().strip(), + "cache_conditions": "fresh app/Git caches per round, warm V2/V3; OS cache uncontrolled; side order alternates", + "fixture_content": "distinct deterministic 32-byte blocks repeated to the configured file size; synthetic compressible source-like bytes", + "git_baseline": "cold V1 depth=1 fetch, ordinary incremental V2/V3 in the same worktree; core.fsync=all + fsync of Git store and worktree files/directories", + "started_utc": datetime.now(timezone.utc).isoformat(), "max_wall_seconds": min(options.deadline_seconds, 14400)}) + for round_number in range(1, options.rounds + 1): + deadline = min(budget.round_deadline(round_number), measurement_limit) + group = root / f"round-{round_number:02}" + group.mkdir() + git_store, git_worktree = group / "git.git", group / "git-worktree" + command(["git", "init", "--bare", str(git_store)], deadline) + old = [] + for version in ("v1", "v2", "v3"): + driver_binding(options) + if service_binding(options) != owner or tip() != current: + raise AssertionError("service/target identity changed before the next publication") + with phase("fixture_and_git_oracle"): + commit, tree = create_version(fixture, round_number, version, options.profile == "smoke", deadline) + expected = expected_manifest(fixture, commit, deadline) + expected_path = group / f"{version}-expected.json" + expected_path.write_text(json.dumps(expected)) + publish_start = time.monotonic() + with phase("git_publication_push"): + git(fixture, deadline, "push", "--no-thin", options.git_url, + f"{commit}:refs/heads/main", env=git_env) + push_ms = (time.monotonic() - publish_start) * 1000 + with phase("updated_publication_identity"): + previous_identity = identity + identity = validate_identity(query(IDENTITY_SQL, deadline), commit, tree, options.database) + native = None + if options.publication_mode == "native": + native = query(NATIVE_SQL, deadline) + validate_native(native, identity, options.instance_id, True, previous_identity) + visible_ms = (time.monotonic() - publish_start) * 1000 + current = commit + driver_env = clean_env({"M2_BASE": options.base_url, "M2_SCOPE": "/project", + "M2_STORE_ROOT": str(group / "scorpio-cache"), + "M2_EXPECTED_VIEW": identity["namespace_view_id"], + "M2_EXPECTED_INSTANCE": options.instance_id, + "M2_TOKEN": os.environ.get("M2_TOKEN", "")}) + if native: + driver_env["M2_EXPECTED_SEQUENCE"] = str(native["sequence"]) + logical_id = f"mst2:{run_id}:r{round_number}:{version}:resolve" if projection else None + if projection: + driver_env["M2_RESOLVE_TRACE_ID"] = logical_id + def scorpio_sync(): + with phase("scorpio_sync"): + got = json.loads(command([str(options.driver), "sync", str(expected_path)], + deadline, env=driver_env)) + source_digest = "sha256:" + hashlib.sha256((checkout / "examples/mst2_update_measure.rs").read_bytes()).hexdigest() + if got.get("driver_source_digest") != source_digest: + raise AssertionError("measurement binary was compiled from different driver source") + return got + def scorpio_oracle(got): + with phase("new_complete_view_audit"): + command([str(options.driver), "audit", str(expected_path), + got["store"], got["content_store"]], deadline) + def scorpio(): + return durable_verified(scorpio_sync, scorpio_oracle) + def git_sync(): + start = time.monotonic() + durable_git = ["git", "-c", "core.fsync=all", "-c", "core.fsyncMethod=fsync", + "--git-dir", str(git_store)] + # The compared service is a fixed current snapshot. A fresh + # full-history fetch would unfairly add earlier repeat bodies + # to Git's cold V1, while MST/2 does not fetch old snapshots. + # Mega2 rejects deepen+have; cold V1 alone needs depth=1. + # Ordinary V2/V3 stop at the preceding fetched commit. + depth = ["--depth=1"] if version == "v1" else [] + with phase("git_baseline_fetch"): + command(durable_git + ["fetch", *depth, "--no-tags", options.git_url, "refs/heads/main"], deadline, env=git_env) + fetched = command(["git", "--git-dir", str(git_store), "rev-parse", "FETCH_HEAD"], deadline).decode().strip() + if fetched != commit: + raise AssertionError("Git comparison fetched a different commit") + fetch_ms = (time.monotonic() - start) * 1000 + with phase("git_baseline_worktree"): + if version == "v1": + command(durable_git + ["worktree", "add", "--detach", str(git_worktree), commit], deadline) + else: + git(git_worktree, deadline, "-c", "core.fsync=all", "-c", "core.fsyncMethod=fsync", "reset", "--hard", commit) + ready_ms = (time.monotonic() - start) * 1000 + with phase("git_baseline_flush"): + flushed_store = fsync_tree(git_store, deadline) + flushed_worktree = fsync_tree(git_worktree, deadline) + # Directory entries for both trees must survive publication. + fd = os.open(group, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + return {"fetch_ms": fetch_ms, "worktree_ready_ms": ready_ms, + "durable_complete_ms": (time.monotonic() - start) * 1000, + "git_store_flush": flushed_store, "worktree_flush": flushed_worktree} + def git_oracle(_): + with phase("git_byte_oracle"): + verify_worktree(git_worktree, expected, deadline) + def baseline(): + return durable_verified(git_sync, git_oracle) + # Alternate side order across repeat/scenario to limit systematic + # advantage from HTTP/object/page-cache warming on the server. + side_order = "scorpio-first" if (round_number + int(version[1])) % 2 == 0 else "git-first" + if side_order == "scorpio-first": + measured, git_measured = scorpio(), baseline() + else: + git_measured, measured = baseline(), scorpio() + trace = None + if projection: + # All timed side operations and immediate byte oracles ended. + # Durable trace parsing/waiting still consumes this round's wall budget. + with phase("projection_trace_collection"): + trace = collector.collect(measured, native, identity, logical_id, deadline) + # Each side's new-view byte oracle ran immediately within that + # side's verified timer. Old-view checks are separate wall time. + with phase("old_complete_view_audit"): + for old_path, old_store, old_content in old: + command([str(options.driver), "audit", str(old_path), old_store, old_content], deadline) + if version == "v2" and measured["fetched_content_units"] != 1: + raise AssertionError("single-file update must fetch exactly one new content unit") + if version == "v3" and measured["fetched_content_units"] != 0: + raise AssertionError("unchanged subtree rename must fetch no content") + after = validate_identity(query(IDENTITY_SQL, deadline), commit, tree, options.database) + if after != identity or tip() != commit or service_binding(options) != owner: + raise AssertionError("fixed service/commit changed across update timing") + old.append((expected_path, measured["store"], measured["content_store"])) + record = {"record": "round", "round": round_number, "version": version, + "fixed_commit": commit, "identity": identity, "git_push_ms": push_ms, + "git_ref_visible_ms": visible_ms, + "publication_visible_ms": visible_ms if native else None, + "publication_and_client_metadata_ready_ms": visible_ms + measured["metadata_ready_ms"] if native else None, + "publication_and_client_durable_complete_ms": visible_ms + measured["durable_complete_ms"] if native else None, + "publication_and_client_scope": "sum of publication observation and client segments; excludes interleaved Git baseline and harness oracle setup", + "publication_timing_scope": "push start through read-only DB certificate observation; an upper bound, not internal server projection duration", + "native_publication": native, "publication_mode": options.publication_mode, + "server_projection": trace, + "server_projection_rebuilt_pages": None, "server_projection_reused_pages": None, + "server_projection_stats": "typed directory-root work; codec-internal radix work is NOT_EXPOSED" if projection else "NOT_EXPOSED", + "durable_verified_scope": "each side operation through its immediate independent full-byte oracle; old-view audits excluded", + "side_order": side_order, "scorpio": measured, "git": git_measured, + "correctness": "PASS", "old_local_complete_views_equal": True, + "old_live_reader_or_fuse_lease": "NOT_RUN", + "fuse_mount": "NOT_RUN"} + records.append(record) + emit(record) + if time.monotonic() >= deadline: + raise TimeoutError("complete round exceeded its fixed budget") + if len(records) != options.rounds * 3: + raise AssertionError("all requested complete V1/V2/V3 rounds are required") + deadline = budget.report_deadline() + projection_writer_status = None + if projection: + with phase("projection_trace_finalization"): + options.finalize_projection(deadline) + projection_writer_status = collector.finish(options.rounds * 3, deadline) + for version in ("v1", "v2", "v3"): + if time.monotonic() >= deadline: + raise TimeoutError("summary exceeded its fixed report budget") + samples = [r for r in records if r["version"] == version] + summary = {"record": "summary", "version": version, "samples": len(samples)} + for metric, select in { + "publication_visible_ms": lambda r: r["publication_visible_ms"], + "publication_and_client_metadata_ready_ms": lambda r: r["publication_and_client_metadata_ready_ms"], + "publication_and_client_durable_complete_ms": lambda r: r["publication_and_client_durable_complete_ms"], + "resolve_ms": lambda r: r["scorpio"]["resolve_ms"], + "metadata_ready_ms": lambda r: r["scorpio"]["metadata_ready_ms"], + "durable_complete_ms": lambda r: r["scorpio"]["durable_complete_ms"], + "durable_verified_ms": lambda r: r["scorpio"]["durable_verified_ms"], + "git_durable_verified_ms": lambda r: r["git"]["durable_verified_ms"], + "git_durable_complete_ms": lambda r: r["git"]["durable_complete_ms"]}.items(): + values = [select(r) for r in samples if select(r) is not None] + summary[metric] = {"p50": percentile(values, 50), "p95": percentile(values, 95)} if values else None + if projection: + for metric in ["projection_elapsed_micros", *sorted(projection_module.WORK_FIELDS)]: + values = [r["server_projection"]["payload"][metric] for r in samples] + summary[metric] = {"p50": percentile(values, 50), "p95": percentile(values, 95)} + emit(summary) + emit({"record": "complete", "round_scenarios": len(records), + "projection_writer_status": projection_writer_status, + "elapsed_seconds": time.monotonic() - started, "correctness": "PASS"}) + + +def parser(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--execute", action="store_true") + p.add_argument("--isolated-deployment", action="store_true") + p.add_argument("--base-url", required=True) + p.add_argument("--git-url", required=True) + p.add_argument("--database", required=True) + p.add_argument("--instance-id", required=True) + p.add_argument("--expect-initial-commit", required=True) + p.add_argument("--service-pid", type=int, required=True) + p.add_argument("--driver", type=Path, required=True) + p.add_argument("--driver-sha256", required=True) + p.add_argument("--run-root", type=Path, required=True) + p.add_argument("--publication-mode", choices=("native", "on-demand"), default="native") + p.add_argument("--projection-traces", action="store_true") + p.add_argument("--profile", choices=("medium", "smoke"), default="medium") + p.add_argument("--rounds", type=int, choices=range(3, 11), default=3) + p.add_argument("--deadline-seconds", type=int, choices=range(60, 14401), default=14400) + p.add_argument("--session-deadline-utc") + p.add_argument("--work-cleanup-deadline-monotonic", type=float, + default=os.environ.get("MST2_WORK_CLEANUP_DEADLINE_MONOTONIC")) + return p + + +if __name__ == "__main__": + opts = parser().parse_args() + if not opts.execute: + endpoint_pair(opts.base_url, opts.git_url) + print(json.dumps({"execute": False, "profile": opts.profile, "rounds": opts.rounds, + "scenarios": ["v1-cold", "v2-single-file", "v3-subtree-rename"], + "max_wall_seconds": min(opts.deadline_seconds, 14400), + "service_or_resource_changes": False, + "git_baseline": "cold V1 depth=1 fetch, ordinary incremental V2/V3 fetch/reset in the same worktree, then explicit fsync", + "publication_mode": opts.publication_mode}, indent=2)) + else: + try: + execute(opts) + except Exception as error: + print(json.dumps(failure_record(error)), file=sys.stderr) + raise SystemExit(1) diff --git a/tests/mst2-e2e/commit_update_budget.py b/tests/mst2-e2e/commit_update_budget.py new file mode 100644 index 00000000..12117146 --- /dev/null +++ b/tests/mst2-e2e/commit_update_budget.py @@ -0,0 +1,344 @@ +"""One immutable session budget for builds, owned setup, rounds and cleanup.""" + +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + + +EXTERNAL_RESERVE = 15 * 60 +CLEANUP_RESERVE = 10 * 60 +REPORT_RESERVE = 10 * 60 +MARGIN = 10 * 60 +ROUND_SECONDS = 25 * 60 +STAGES = {"server-build": 35 * 60, "client-build": 20 * 60, + "fences": 10 * 60, "setup": 10 * 60} + + +def utc(value): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.utcoffset().total_seconds() != 0: + raise ValueError("session deadline must be explicit UTC") + return parsed + + +class SessionBudget: + def __init__(self, deadline_utc, rounds, cleanup_deadline=None): + absolute = utc(deadline_utc) + if isinstance(rounds, bool) or not 3 <= rounds <= 10: + raise ValueError("at least three complete rounds are required") + # Workflow establishes this monotonic anchor once, before builds. A + # standalone invocation also converts its absolute UTC deadline once. + if cleanup_deadline is None: + remaining = absolute.timestamp() - time.time() + if not 0 < remaining <= 235 * 60: + raise ValueError("session deadline exceeds the fixed window") + cleanup_deadline = time.monotonic() + remaining - EXTERNAL_RESERVE + if (not math.isfinite(cleanup_deadline) + or cleanup_deadline - time.monotonic() > 220 * 60): + raise ValueError("invalid work/cleanup deadline anchor") + self.deadline_utc = absolute.isoformat() + self.cleanup_deadline = cleanup_deadline + self.measurement_deadline = cleanup_deadline - CLEANUP_RESERVE - REPORT_RESERVE + self.rounds = rounds + + def require(self, seconds): + if self.cleanup_deadline - time.monotonic() < seconds: + raise TimeoutError("insufficient complete-stage session budget") + + def stage_deadline(self, stage): + if stage not in STAGES: + raise ValueError("unknown session stage") + names = tuple(STAGES) + later = sum(STAGES[name] for name in names[names.index(stage) + 1:]) + reserve = later + self.rounds * ROUND_SECONDS + REPORT_RESERVE + CLEANUP_RESERVE + MARGIN + self.require(STAGES[stage] + reserve) + return min(time.monotonic() + STAGES[stage], self.cleanup_deadline - reserve) + + def round_deadline(self, number): + if not 1 <= number <= self.rounds: + raise ValueError("invalid round number") + reserve = ((self.rounds - number) * ROUND_SECONDS + + REPORT_RESERVE + CLEANUP_RESERVE + MARGIN) + self.require(ROUND_SECONDS + reserve) + return min(time.monotonic() + ROUND_SECONDS, self.cleanup_deadline - reserve) + + def report_deadline(self): + self.require(REPORT_RESERVE + CLEANUP_RESERVE) + return min(time.monotonic() + REPORT_RESERVE, + self.cleanup_deadline - CLEANUP_RESERVE) + + +def from_options(options): + existing = getattr(options, "budget", None) + if existing is not None: + return existing + return SessionBudget(options.session_deadline_utc, options.rounds, + getattr(options, "work_cleanup_deadline_monotonic", None)) + + +def group_members(pgid, started): + """Only this new-session group; zombies hold no executing work.""" + members = [] + for proc in Path("/proc").iterdir(): + if not proc.name.isdecimal(): + continue + try: + fields = (proc / "stat").read_text().rsplit(") ", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): + continue + if int(fields[2]) != pgid: + continue + if (int(fields[3]) != pgid + or (started is not None and int(fields[19]) < int(started))): + raise AssertionError("owned process group identity changed") + if started is not None and int(proc.name) == pgid and fields[19] != str(started): + raise AssertionError("owned group leader identity was replaced") + if fields[0] not in ("Z", "X"): + members.append(int(proc.name)) + return members + + +def reap_owned(process, deadline): + release = getattr(process, "release_reap", None) + if release is not None: + release() + process.wait(timeout=max(0, deadline - time.monotonic())) + + +def reap_exited(process): + """No signals or added wait budget: only reap an already exited child.""" + if process is None or process.returncode is not None or sys.platform != "linux": + return + observed = os.waitid(os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if observed is not None: + release = getattr(process, "release_reap", None) + if release is not None: + release() + process.wait(timeout=0) + + +@contextmanager +def cleanup_reap(process): + """The final reap runs after all owned-group signal paths have ended.""" + try: + yield + finally: + failed = sys.exc_info()[0] is not None + try: + reap_exited(process) + except BaseException: + # Keep the original cleanup/timeout/interrupt failure. This final + # nonblocking reap cannot convert an incomplete cleanup to PASS. + if not failed: + raise + + +def stop_group(pgid, started, deadline, process=None): + """Reaped leaders do not prove that their owned descendants exited.""" + with cleanup_reap(process): + if group_members(pgid, started): + try: + os.killpg(pgid, signal.SIGTERM) + except ProcessLookupError: + pass + term_until = min(deadline, time.monotonic() + 5) + while group_members(pgid, started) and time.monotonic() < term_until: + time.sleep(min(.05, max(0, term_until - time.monotonic()))) + if group_members(pgid, started): + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + while group_members(pgid, started) and time.monotonic() < deadline: + time.sleep(min(.01, max(0, deadline - time.monotonic()))) + if group_members(pgid, started): + raise TimeoutError("owned group remained active at its original deadline") + if process is not None: + reap_owned(process, deadline) + if time.monotonic() >= deadline: + raise TimeoutError("owned group verification exceeded original deadline") + + +def process_start(pid): + if sys.platform == "linux": + return Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1].split()[19] + return None + + +class PinnedProcess(subprocess.Popen): + """Observe Linux exit without freeing the PID until group cleanup finishes. + + communicate() calls wait(), including when no pipes are captured. An + ordinary wait would release the leader PID before descendant cleanup and + permit an unrelated new session to reuse its process-group number. + """ + def __init__(self, *args, **kwargs): + self.reap_pinned = True + super().__init__(*args, **kwargs) + + def release_reap(self): + self.reap_pinned = False + + def _wait(self, timeout=None): + # CPython communicate's KeyboardInterrupt handler calls _wait directly. + if self.reap_pinned: + return self.wait(timeout=timeout) + return super()._wait(timeout=timeout) + + def wait(self, timeout=None): + if not self.reap_pinned: + return super().wait(timeout=timeout) + if self.returncode is not None: + raise AssertionError("owned command leader was reaped before group cleanup") + until = None if timeout is None else time.monotonic() + timeout + while True: + observed = os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if observed is not None: + if observed.si_code == os.CLD_EXITED: + return observed.si_status + return -observed.si_status + if until is not None and time.monotonic() >= until: + raise subprocess.TimeoutExpired(self.args, timeout) + pause = .01 if until is None else min(.01, max(0, until - time.monotonic())) + time.sleep(pause) + + +def abort_startup(process, deadline): + """A direct unreaped Popen child pins its PID while startup is rejected.""" + if process.returncode is not None: + raise AssertionError("startup abort requires an unreaped owned child") + try: + pgid, sid = os.getpgid(process.pid), os.getsid(process.pid) + except ProcessLookupError: + # No leader is distinct from an unreadable/replaced leader. Do not + # signal an unbound group; still reap this direct child and fail closed + # if any active group remains. + if group_members(process.pid, None): + raise AssertionError("missing startup leader still has an unbound group") from None + reap_owned(process, deadline) + if time.monotonic() >= deadline: + raise TimeoutError("startup reap exceeded original deadline") + return False + if pgid != process.pid or sid != process.pid: + raise AssertionError("startup child is outside its owned new-session group") + # No poll/communicate/wait has released this direct child PID. Group scans + # and signals finish before stop_group reaps it, even if its leader exits. + stop_group(process.pid, None, deadline, process) + return True + + +def run_process(args, deadline, env=None, data=None, capture=True): + """Terminate and reap the owned child group within this same deadline.""" + now = time.monotonic() + remaining = deadline - now + if remaining <= 0: + raise TimeoutError("operation deadline reached before child startup") + reserve = min(10.0, remaining / 4) + run_until = deadline - reserve + popen = PinnedProcess if sys.platform == "linux" else subprocess.Popen + process = popen(args, stdin=subprocess.PIPE if data is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + env=env, start_new_session=True) + started = None + + def signal_group(signum): + if process.returncode is not None: + raise AssertionError("refusing signal after owned command leader was reaped") + # The unreaped direct child pins this number through scan and signal. + # Check the recorded identity before every signal, including KILL. + if not group_members(process.pid, started): + return + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + pass + + def terminate(): + if started is None: + if process.returncode is None: + abort_startup(process, deadline) + return + signal_group(signal.SIGTERM) + killed = False + term_until = min(deadline, time.monotonic() + reserve / 2) + try: + process.communicate(timeout=max(0, term_until - time.monotonic())) + except subprocess.TimeoutExpired: + signal_group(signal.SIGKILL) + killed = True + try: + process.communicate(timeout=max(0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + # Do not grant a fresh wait timeout when a pipe remains open. + for pipe in (process.stdin, process.stdout, process.stderr): + if pipe is not None: + pipe.close() + raise TimeoutError("owned child did not reap within its deadline") from None + # Descendants may ignore TERM after their leader already exited. + if not killed: + signal_group(signal.SIGKILL) + if started is not None: + stop_group(process.pid, started, deadline, process) + + with cleanup_reap(process): + try: + started = process_start(process.pid) + except FileNotFoundError: + if abort_startup(process, deadline): + raise AssertionError("startup process identity could not be established") from None + except BaseException: + abort_startup(process, deadline) + raise + try: + output, error = process.communicate(data, timeout=max(0, run_until - time.monotonic())) + except subprocess.TimeoutExpired: + terminate() + raise TimeoutError("owned child exceeded its operation budget") from None + except BaseException: + terminate() + raise + # A short-lived command owns its new-session group even when descendants + # close inherited pipes and its leader returns a successful exit status. + if started is not None: + stop_group(process.pid, started, deadline, process) + if time.monotonic() >= deadline: + raise TimeoutError("owned child verification exceeded original deadline") + return process.returncode, output, error + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--session-deadline-utc", default=os.environ.get("MST2_SESSION_DEADLINE"), required=False) + parser.add_argument("--work-cleanup-deadline-monotonic", type=float, + default=os.environ.get("MST2_WORK_CLEANUP_DEADLINE_MONOTONIC")) + parser.add_argument("--rounds", type=int, choices=range(3, 11), default=3) + parser.add_argument("--stage", choices=tuple(STAGES), required=True) + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args() + if not options.session_deadline_utc: + raise ValueError("an original absolute session deadline is required") + args = options.command + if args and args[0] == "--": + args = args[1:] + if not args: + raise ValueError("a stage command is required") + deadline = from_options(options).stage_deadline(options.stage) + status, _, _ = run_process(args, deadline, capture=False) + return status + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (Exception, KeyboardInterrupt) as error: + print(json.dumps({"record": "session_budget_failure", "error_type": type(error).__name__}), flush=True) + raise SystemExit(1) diff --git a/tests/mst2-e2e/commit_update_ci.py b/tests/mst2-e2e/commit_update_ci.py new file mode 100644 index 00000000..134f0669 --- /dev/null +++ b/tests/mst2-e2e/commit_update_ci.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""Opt-in isolated setup for the real benchmark on disposable hosted GitHub runners. + +Plan-only by default. Resources are owned by this job's unique Compose project. +No existing server, cloud resource or persistent deployment is accepted. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import signal +import socket +import subprocess +import sys +import time +from urllib.request import urlopen +import uuid + +import commit_update_bench as bench +import commit_update_budget as budget_module +import commit_update_projection as projection_module + + +def hosted_root(root): + if (sys.platform != "linux" or os.environ.get("GITHUB_ACTIONS") != "true" + or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted"): + raise ValueError("execution is limited to a disposable GitHub-hosted Linux runner") + run, attempt = os.environ.get("GITHUB_RUN_ID", ""), os.environ.get("GITHUB_RUN_ATTEMPT", "") + if not run.isdecimal() or not attempt.isdecimal(): + raise ValueError("job ownership identifiers are missing") + expected = Path(os.environ["RUNNER_TEMP"]).resolve(strict=True) / f"mst2-real-{run}-{attempt}" + if root.absolute() != expected or root.is_symlink(): + raise ValueError("owned root must be the exact unique job directory under RUNNER_TEMP") + return expected, f"m2perf-{run}-{attempt}" + + +def toml(data): + """Serialize the trusted server template, including its token-table array.""" + lines = [] + def value(item): + if isinstance(item, bool): + return "true" if item else "false" + if isinstance(item, (str, int)): + return json.dumps(item, ensure_ascii=False) + if isinstance(item, list): + return "[" + ", ".join(value(x) for x in item) + "]" + raise TypeError("unsupported template value") + def table(items, path, array=False): + if path: + name = ".".join(path) + lines.append(("[[" + name + "]]" if array else "[" + name + "]")) + for key, item in items.items(): + if not re.fullmatch(r"[A-Za-z0-9_]+", key): + raise ValueError("unexpected template key") + if not isinstance(item, dict) and not (isinstance(item, list) and item and isinstance(item[0], dict)): + lines.append(key + " = " + value(item)) + for key, item in items.items(): + if isinstance(item, dict): + table(item, path + [key]) + elif isinstance(item, list) and item and isinstance(item[0], dict): + for entry in item: + table(entry, path + [key], True) + table(data, []) + return "\n".join(lines) + "\n" + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def initialize_owned_native(database, instance, env, deadline): + """Maintenance bootstrap for this job's fresh DB, before starting writers. + + service init does not invoke Mega2's maintenance-only native initializer. + Match its root/epoch/sequence contract, with a stricter empty-fixture gate. + This does not create a READY certificate; the first real push must do that. + """ + if (not re.fullmatch(r"mst2_bench_[0-9a-f]{32}", database) + or env.get("PGDATABASE") != database + or env.get("PGHOST") != "127.0.0.1"): + raise ValueError("native bootstrap requires the exact newly owned loopback database") + if str(uuid.UUID(instance)) != instance: + raise ValueError("native bootstrap requires a canonical instance UUID") + sql = f"""BEGIN; +SET LOCAL search_path = public; +SELECT pg_advisory_xact_lock(1297043024, 1229867349); +DO $owned_native$ +DECLARE root mega_refs%ROWTYPE; +BEGIN + IF current_database() <> '{database}' THEN + RAISE EXCEPTION 'owned database differs'; + END IF; + IF (SELECT count(*) FROM queue_control WHERE id=1) <> 1 THEN + RAISE EXCEPTION 'fresh queue control is missing'; + END IF; + PERFORM id FROM queue_control WHERE id=1 FOR UPDATE; + IF EXISTS (SELECT 1 FROM push_queue) + OR EXISTS (SELECT 1 FROM mst2_native_head) + OR EXISTS (SELECT 1 FROM mst2_native_publication) + OR EXISTS (SELECT 1 FROM mst2_namespace_seq) + OR EXISTS (SELECT 1 FROM mst2_publication) + OR EXISTS (SELECT 1 FROM mst2_publication_outbox) + OR EXISTS (SELECT 1 FROM mst2_queue_noop_receipt) THEN + RAISE EXCEPTION 'native bootstrap requires an unused publication fixture'; + END IF; + SELECT * INTO STRICT root FROM mega_refs + WHERE path='/' AND ref_name='refs/heads/main' AND NOT is_cl FOR UPDATE; + IF root.ref_commit_hash !~ '^[0-9a-f]{{40}}$' + OR root.ref_tree_hash !~ '^[0-9a-f]{{40}}$' + OR (SELECT count(*) FROM mega_commit + WHERE commit_id=root.ref_commit_hash AND tree=root.ref_tree_hash) <> 1 + OR (SELECT count(*) FROM mega_tree WHERE tree_id=root.ref_tree_hash) <> 1 THEN + RAISE EXCEPTION 'fresh native root is missing or mismatched'; + END IF; + INSERT INTO mst2_native_head + (namespace,instance_id,sequence,writer_epoch,root_commit,root_tree,state) + VALUES ('/','{instance}',0,1,root.ref_commit_hash,root.ref_tree_hash,'INITIALIZING'); +END $owned_native$; +COMMIT;""" + bench.command(["psql", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + deadline, env=env, data=sql.encode()) + # Independently read back the actual stored root and initial head before + # any writer is started. No verified object or publication is fabricated. + rows = bench.query(bench.IDENTITY_SQL, deadline, env=env) + # service init stores only the global ref. The Git HTTP route lazily + # materializes /project; its full identity fence still runs after ls-remote. + if not isinstance(rows, list): + raise AssertionError("fresh global root ref is missing") + roots = [row for row in rows if isinstance(row, dict) and row.get("path") == "/"] + if len(roots) != 1: + raise AssertionError("exactly one fresh global main ref is required") + root = roots[0] + if (root["database"] != database or root["commit_tree"] != root["tree"] + or not re.fullmatch(r"[0-9a-f]{40}", root["commit"]) + or not re.fullmatch(r"[0-9a-f]{40}", root["tree"])): + raise AssertionError("fresh global database or stored commit/tree differs") + raw = bytes.fromhex(root["raw_tree"]) + if hashlib.sha1(b"tree " + str(len(raw)).encode() + b"\0" + raw).hexdigest() != root["tree"]: + raise AssertionError("fresh global raw tree does not match its Git object ID") + identity = {"global_commit": root["commit"], "global_tree": root["tree"]} + native = bench.query(bench.NATIVE_SQL, deadline, env=env) + if (not native or native["sequence"] != 0 or native["state"] != "INITIALIZING" + or native["certificate_receipt_id"] is not None): + raise AssertionError("fresh native head is not the initial epoch/sequence") + bench.validate_native(native, identity, instance, False) + return {"record": "owned_native_initialization", "instance_id": instance, + "sequence": 0, "writer_epoch": 1, "state": "INITIALIZING", + "global_commit": identity["global_commit"], + "global_tree": identity["global_tree"], "correctness": "PASS", + "production_service_init_wired": False} + + +def dependencies(source, project, ports, deadline): + raw = bench.command(["docker", "compose", "-f", str(source / "docker/docker-compose.test.yml"), + "config", "--format", "json"], deadline) + config = json.loads(raw) + selected = {name: config["services"][name] for name in ("postgres", "redis", "rustfs", "rustfs-init")} + for name, service in selected.items(): + if "build" in service or "volumes" in service or "container_name" in service: + raise AssertionError("dependency template unexpectedly escapes disposable job storage") + service["networks"] = {"default": None} + service.pop("ports", None) + if name in ports: + target = {"postgres": 5432, "redis": 6379, "rustfs": 9000}[name] + service["ports"] = [{"target": target, "published": str(ports[name]), + "host_ip": "127.0.0.1", "protocol": "tcp"}] + return {"services": selected, "networks": {"default": {"name": project + "-network"}}} + + +def stop_owned(root, project, deadline, process=None): + state_path = root / "owned.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text()) + if state["project"] != project: + raise AssertionError("cleanup ownership differs from this job") + compose = root / "dependencies.json" + if hashlib.sha256(compose.read_bytes()).hexdigest() != state["compose_sha256"]: + raise AssertionError("cleanup Compose configuration differs from owned startup") + service = state.get("service") + if service: + pid = service["pid"] + if service.get("pgid") != pid or service.get("sid") != pid: + raise AssertionError("owned new-session group binding is missing") + proc = Path(f"/proc/{pid}") + reaped = process is not None and process.returncode is not None + if not reaped and proc.exists(): + fields = proc.joinpath("stat").read_text().rsplit(") ", 1)[1].split() + started = fields[19] + argv = [x.decode() for x in proc.joinpath("cmdline").read_bytes().split(b"\0") if x] + if (started != service["starttime"] + or (fields[0] not in ("Z", "X") + and argv[:3] != [state["binary"], "--config", str(root / "service.toml")]) + or os.getpgid(pid) != pid or os.getsid(pid) != pid): + raise AssertionError("refusing cleanup of a replaced or unrelated service") + elif not reaped and not proc.exists() and budget_module.group_members(pid, service["starttime"]): + raise AssertionError("refusing to signal a group whose owned leader is missing") + budget_module.stop_group(pid, service["starttime"], deadline, process) + bench.command(["docker", "compose", "-p", project, "-f", str(compose), "down", "--volumes"], deadline) + containers = bench.command(["docker", "ps", "-aq", "--filter", + "label=com.docker.compose.project=" + project], deadline) + network = bench.command(["docker", "network", "ls", "-q", "--filter", + "name=^" + project + "-network$"], deadline) + if containers.strip() or network.strip(): + raise AssertionError("owned cleanup left project containers or network") + if time.monotonic() >= deadline: + raise TimeoutError("owned cleanup inventory exceeded original deadline") + state.pop("service", None) + state_path.write_text(json.dumps(state)) + if time.monotonic() >= deadline: + raise TimeoutError("owned cleanup metadata exceeded original deadline") + print(json.dumps({"record": "owned_cleanup", "project": project, "correctness": "PASS"}), flush=True) + + +def owned_service_exit(process): + """Observe readiness/exit without releasing the owned leader's PID.""" + if process.returncode is not None or not isinstance(process, budget_module.PinnedProcess): + raise AssertionError("owned service leader must remain pinned") + observed = os.waitid(os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + if observed is None: + return None + return observed.si_status if observed.si_code == os.CLD_EXITED else -observed.si_status + + +def graceful_owned(root, project, deadline, process): + """Only this newly owned service group; final fallback retains the same H.""" + state_path = root / "owned.json" + state = json.loads(state_path.read_text()) + service = state.get("service", {}) + if (state["project"] != project or service.get("pid") != process.pid + or service.get("pgid") != process.pid or service.get("sid") != process.pid + or hashlib.sha256((root / "dependencies.json").read_bytes()).hexdigest() != state["compose_sha256"]): + raise AssertionError("graceful stop ownership differs from this job") + proc = Path(f"/proc/{process.pid}") + fields = proc.joinpath("stat").read_text().rsplit(") ", 1)[1].split() + argv = [x.decode() for x in proc.joinpath("cmdline").read_bytes().split(b"\0") if x] + if (fields[19] != service["starttime"] or fields[0] in ("Z", "X") + or argv[:3] != [state["binary"], "--config", str(root / "service.toml")] + or os.getpgid(process.pid) != process.pid or os.getsid(process.pid) != process.pid + or owned_service_exit(process) is not None): + raise AssertionError("refusing graceful stop of a changed or exited service") + if time.monotonic() >= deadline: + raise TimeoutError("no original budget remains for projection drain") + os.killpg(process.pid, signal.SIGINT) + until = min(deadline, time.monotonic() + 7) + while budget_module.group_members(process.pid, service["starttime"]) and time.monotonic() < until: + time.sleep(min(.01, max(0, until - time.monotonic()))) + if (budget_module.group_members(process.pid, service["starttime"]) + or owned_service_exit(process) != 0 or time.monotonic() >= deadline): + raise TimeoutError("owned service did not complete its bounded graceful drain") + budget_module.reap_owned(process, deadline) + state.pop("service") + state_path.write_text(json.dumps(state)) + + +def execute(options): + root, project = hosted_root(options.run_root) + if root.exists(): + raise ValueError("disposable job directory must not already exist") + if not re.fullmatch(r"[0-9a-f]{40}", options.mega_sha): + raise ValueError("server source must be an immutable full SHA-1") + budget = budget_module.from_options(options) + if getattr(options, "projection_traces", False): + original = projection_module.window_anchor(options.session_started_utc, options.session_deadline_utc) + if budget.cleanup_deadline > original + 1: + raise ValueError("projection work deadline exceeds its original dispatch anchor") + deadline = budget.stage_deadline("setup") + source = options.mega_source.resolve(strict=True) + source_sha = bench.git(source, deadline, "rev-parse", "HEAD").decode().strip() + if source_sha != options.mega_sha or bench.git(source, deadline, "status", "--porcelain").strip(): + raise AssertionError("server checkout differs from the reviewed immutable source") + binary = options.mega_binary.resolve(strict=True) + if not binary.is_relative_to(source / "target"): + raise ValueError("server binary must be built inside the immutable server checkout") + bench.driver_binding(options) + ports = {name: free_port() for name in ("postgres", "redis", "rustfs", "http")} + if len(set(ports.values())) != 4: + raise AssertionError("ephemeral ports collided before resource creation") + compose = dependencies(source, project, ports, deadline) + existing = bench.command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], deadline) + existing_network = bench.command(["docker", "network", "ls", "-q", "--filter", "name=^" + project + "-network$"], deadline) + if existing.strip() or existing_network.strip(): + raise AssertionError("refusing to reuse or clean up an existing Compose project/network") + root.mkdir(mode=0o700) + compose_path = root / "dependencies.json" + compose_path.write_text(json.dumps(compose)) + state = {"project": project, "binary": str(binary), "mega_source_sha": source_sha, + "compose_sha256": hashlib.sha256(compose_path.read_bytes()).hexdigest()} + state_path = root / "owned.json" + state_path.write_text(json.dumps(state)) + process = None + log = None + try: + bench.command(["docker", "compose", "-p", project, "-f", str(compose_path), + "up", "-d", "--wait", "--wait-timeout", "180"], min(deadline, time.monotonic() + 240)) + db = "mst2_bench_" + uuid.uuid4().hex + env = bench.clean_env({"PGHOST": "127.0.0.1", "PGPORT": str(ports["postgres"]), + "PGUSER": "mega2", "PGPASSWORD": "mega2_test_password", "PGDATABASE": "mega2"}) + bench.command(["psql", "-X", "-v", "ON_ERROR_STOP=1", "-c", "CREATE DATABASE " + db], deadline, env=env) + env["PGDATABASE"] = db + git_token, token = secrets.token_urlsafe(32), secrets.token_urlsafe(32) + for name, secret in (("git-token", git_token), ("mst2-token", token)): + print("::add-mask::" + secret, flush=True) + file = root / name + file.write_text(secret) + file.chmod(0o600) + config = bench.tomllib.loads((source / "config/config-storage-only.toml").read_text()) + config["base_dir"] = str(root / "service-data") + config["log"].update(print_std=False, with_ansi=False) + config["database"].update(db_url=f"postgres://mega2:mega2_test_password@127.0.0.1:{ports['postgres']}/{db}", + max_connection=8, min_connection=1, acquire_timeout=60, connect_timeout=30) + config["redis"]["url"] = f"redis://127.0.0.1:{ports['redis']}" + config["monorepo"].update(root_dirs=["third-party", "project"], object_format="sha1", push_policy="trunk") + config["pack"].update(pack_decode_mem_size="512M", pack_decode_cache_path=str(root / "pack-cache")) + config["object_storage"]["s3"].update(endpoint_url=f"http://127.0.0.1:{ports['rustfs']}", bucket="mega2") + config["git"].update(push_auth="token", ssh_receive_pack=False, + push_tokens=[{"name": "owned-benchmark", "token": "${file:" + str(root / "git-token") + "}", + "paths": ["/project"]}]) + instance = str(uuid.uuid4()) + config["mst2"] = {"enabled": True, "instance_uuid": instance, "publication_enabled": True, + "auth_token": "${file:" + str(root / "mst2-token") + "}"} + if getattr(options, "projection_traces", False): + config["mst2"]["projection_observation_enabled"] = True + for name in ("oci", "agent_capture", "storage_events"): + config[name] = {"enabled": False} + config_path = root / "service.toml" + config_path.write_text(toml(config)) + config_path.chmod(0o600) + service_env = bench.clean_env({"MEGA_BASE_DIR": config["base_dir"], "MEGA_CACHE_DIR": str(root / "cache"), + "MEGA_GIT_OBJECT_CACHE_PREFIX": project}) + prefix = [str(binary), "--config", str(config_path)] + bench.command(prefix + ["config", "validate"], deadline, env=service_env) + bench.command(prefix + ["service", "init", "--yes"], deadline, env=service_env) + with bench.phase("owned_native_initialization"): + print(json.dumps(initialize_owned_native(db, instance, env, deadline)), flush=True) + log = (root / "service-private.log").open("wb") + process = budget_module.PinnedProcess(prefix + ["service", "http", "--host", "127.0.0.1", "-p", str(ports["http"])], + stdin=subprocess.DEVNULL, stdout=log, stderr=log, + env=service_env, start_new_session=True) + started = None + try: + started = budget_module.process_start(process.pid) + state["service"] = {"pid": process.pid, "pgid": process.pid, "sid": process.pid, + "starttime": started} + state_path.write_text(json.dumps(state)) + except BaseException: + # Startup identity/state failures must not bypass owned finally. + # The direct Popen child is still unreaped and pins its group ID. + if started is None: + budget_module.abort_startup(process, budget.cleanup_deadline) + else: + budget_module.stop_group(process.pid, started, budget.cleanup_deadline, process) + raise + base = f"http://127.0.0.1:{ports['http']}" + ready_until = min(deadline, time.monotonic() + 180) + while True: + if owned_service_exit(process) is not None or time.monotonic() >= ready_until: + raise RuntimeError("owned service failed readiness") + try: + with urlopen(base + "/api/v2/snapshots/capabilities", + timeout=min(2, max(.001, ready_until - time.monotonic()))) as response: + if response.status == 200: + break + except OSError: + pass + time.sleep(min(.2, max(0, ready_until - time.monotonic()))) + initial = bench.command(["git", "ls-remote", base + "/project", "refs/heads/main"], deadline).decode().split() + if len(initial) != 2 or initial[1] != "refs/heads/main": + raise AssertionError("owned service did not initialize exactly one project main") + env.update(M2_TOKEN=token, M2_GIT_TOKEN=git_token) + budget.require(options.rounds * budget_module.ROUND_SECONDS + + budget_module.REPORT_RESERVE + budget_module.CLEANUP_RESERVE + + budget_module.MARGIN) + print(json.dumps({"record": "owned_server_build", "source_sha": source_sha, + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), "project": project, + "requested_profile": options.profile, "actual_profile": options.profile, + "requested_rounds": options.rounds, "actual_rounds": options.rounds, + "remaining_session_seconds": budget.cleanup_deadline - time.monotonic()}), flush=True) + os.environ.update(env) + args = bench.parser().parse_args([ + "--execute", "--isolated-deployment", "--base-url", base, "--git-url", base + "/project", + "--database", db, "--instance-id", instance, "--expect-initial-commit", initial[0], + "--service-pid", str(process.pid), "--driver", str(options.driver.resolve()), + "--driver-sha256", options.driver_sha256, "--run-root", str(root / "measurements"), + "--profile", options.profile, "--rounds", str(options.rounds), + "--session-deadline-utc", options.session_deadline_utc]) + args.budget = budget + if getattr(options, "projection_traces", False): + args.projection_traces = True + args.finalize_projection = lambda original_deadline: graceful_owned(root, project, original_deadline, process) + if time.monotonic() >= deadline: + raise TimeoutError("owned setup exceeded its fixed stage budget") + with bench.phase("commit_update_benchmark"): + bench.execute(args) + finally: + try: + stop_owned(root, project, budget.cleanup_deadline, process) + finally: + if log: + log.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute", action="store_true") + parser.add_argument("--cleanup", action="store_true") + parser.add_argument("--projection-traces", action="store_true") + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--mega-source", type=Path) + parser.add_argument("--mega-sha") + parser.add_argument("--mega-binary", type=Path) + parser.add_argument("--driver", type=Path) + parser.add_argument("--driver-sha256") + parser.add_argument("--session-deadline-utc") + parser.add_argument("--session-started-utc") + parser.add_argument("--work-cleanup-deadline-monotonic", type=float, + default=os.environ.get("MST2_WORK_CLEANUP_DEADLINE_MONOTONIC")) + parser.add_argument("--profile", choices=("smoke", "medium"), default="medium") + parser.add_argument("--rounds", type=int, choices=range(3, 11), default=3) + opts = parser.parse_args() + try: + if opts.execute and opts.cleanup: + raise ValueError("execute and cleanup are separate operations") + if opts.cleanup: + owned, compose_project = hosted_root(opts.run_root) + if (owned / "owned.json").exists(): + stop_owned(owned, compose_project, budget_module.from_options(opts).cleanup_deadline) + elif not opts.execute: + print(json.dumps({"execute": False, "profile": opts.profile, "rounds": opts.rounds, + "resources": "one unique disposable hosted-runner Compose project; no cloud resources", + "max_session_seconds": 14400, "persistent_service_changes": False})) + else: + required = (opts.mega_source, opts.mega_sha, opts.mega_binary, opts.driver, + opts.driver_sha256, opts.session_deadline_utc) + if not all(required): + raise ValueError("all immutable build and shared deadline arguments are required") + def interrupted(signum, frame): + raise KeyboardInterrupt + signal.signal(signal.SIGTERM, interrupted) + execute(opts) + except (Exception, KeyboardInterrupt) as error: + print(json.dumps(bench.failure_record(error)), file=sys.stderr) + raise SystemExit(1) diff --git a/tests/mst2-e2e/commit_update_projection.py b/tests/mst2-e2e/commit_update_projection.py new file mode 100644 index 00000000..b3a6446d --- /dev/null +++ b/tests/mst2-e2e/commit_update_projection.py @@ -0,0 +1,384 @@ +"""Bounded, fail-closed collection of the actual typed native resolve sink.""" + +import hashlib +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import re +import stat +import struct +import time +import uuid + + +RECORD_BYTES = 32 * 1024 +RECORD_LIMIT = 64 +FILE_BYTES = RECORD_BYTES * RECORD_LIMIT +STATUS_BYTES = 4096 +U64_MAX = (1 << 64) - 1 +WORK_FIELDS = frozenset({ + "directories_rebuilt", "reused_subtree_roots", "directory_root_pages_built", + "directory_root_page_bytes_built", "directory_root_pages_reused", + "directory_root_page_bytes_reused", "directory_entries_scanned", + "scope_path_entries_examined", "tree_fetches", "verified_blob_hits", + "verified_blob_misses", "raw_bytes_fetched", "raw_bytes_hashed", +}) +FIXED = { + "observation_revision": 1, "phase": "resolve_directory_projection", + "source_domain": "native-git", "schema_version": 2, "metadata_codec": 1, + "materialization_policy": 1, "fs_semantics": 1, "access_projection": 0, + "verification_revision": 2, "projection_revision": 1, + "page_counter_scope": "returned-directory-root-pages", "codec_radix_work": "NOT_EXPOSED", + "message": "native resolve directory projection succeeded", +} +PAYLOAD_FIELDS = frozenset(FIXED) | WORK_FIELDS | frozenset({ + "request_id", "instance_id", "root_commit_oid", "root_tree_oid", + "native_certificate_receipt_id", "native_writer_epoch", "native_publication_sequence", + "scope", "namespace_view_id", "snapshot_id", "metadata_root", "projection_elapsed_micros", +}) +STATUS_FIELDS = frozenset({ + "writer_revision", "sink_instance", "accepted_records", "written_sequence", + "written_records", "written_bytes", "rolling_sha256", "first_error_code", "closed", +}) +ENVELOPE_FIELDS = frozenset({ + "writer_revision", "sink_instance", "record_sequence", "payload", "payload_sha256", +}) +RECEIPT_FIELDS = frozenset({ + "logical_request_id", "attempt_ids", "final_attempt_id", "retry_count", +}) + + +class TraceRejected(AssertionError): + """Fixed messages only; never record contents, paths or credentials.""" + + +class AckPending(Exception): + """A valid live sink has not yet acknowledged every accepted record.""" + + +def reject(): + raise TraceRejected("native projection observation rejected") + + +def integer(value, maximum=U64_MAX, positive=False): + if type(value) is not int or not (int(positive) <= value <= maximum): + reject() + return value + + +def shape(value, fields): + if type(value) is not dict or set(value) != fields: + reject() + + +def pairs(items): + result = {} + for key, value in items: + if key in result: + reject() + result[key] = value + return result + + +def parse(raw): + try: + return json.loads(raw, object_pairs_hook=pairs, + parse_constant=lambda _: reject()) + except (ValueError, UnicodeError, RecursionError): + reject() + + +def digest(raw): + return "sha256:" + hashlib.sha256(raw).hexdigest() + + +def canonical_digest(value): + if type(value) is not str or not re.fullmatch(r"sha256:[0-9a-f]{64}", value): + reject() + return bytes.fromhex(value[7:]) + + +def canonical_uuid(value): + try: + result = uuid.UUID(value) + except (ValueError, TypeError, AttributeError): + reject() + if type(value) is not str or str(result) != value or not result.int: + reject() + return result + + +def token(value, maximum=128): + if type(value) is not str or not re.fullmatch(r"[A-Za-z0-9_.:/-]{1," + str(maximum) + "}", value): + reject() + + +def descriptor_bytes(payload): + # This experiment is confined to the explicit /project scope. MSD2 uses + # little-endian u16 fields; independently rederive its canonical bytes. + if payload["scope"] != "/project": + reject() + scope = payload["scope"].encode() + return (b"MSD2" + struct.pack("= started + timedelta(minutes=220) + or admission and now - started > timedelta(minutes=15)): + reject() + return time.monotonic() + (started + timedelta(minutes=220) - now).total_seconds() + + +class ProjectionCollector: + def __init__(self, cache): + self.cache = Path(cache).absolute() + self.directory_bindings = {} + self.record_identity = None + self.allowed = set() + self.finals = {} + self.last_bytes = b"" + self._directory(self.cache, private=False) + logs = self.cache / "logs" + self._directory(logs, private=False) + parent = logs / "mst2-native-projection" + self._directory(parent) + entries = list(parent.iterdir()) + if len(entries) != 1: + reject() + self.root = entries[0] + self.instance = str(canonical_uuid(self.root.name)) + self._directory(self.root) + + def _directory(self, path, private=True): + # Reject symlinks throughout the actual configured anchor chain. + for component in [*reversed(path.parents), path]: + info = component.lstat() + if not stat.S_ISDIR(info.st_mode): + reject() + info = path.lstat() + if private and os.name == "posix" and (info.st_uid != os.geteuid() or stat.S_IMODE(info.st_mode) != 0o700): + reject() + identity = (info.st_dev, info.st_ino) + if path in self.directory_bindings and self.directory_bindings[path] != identity: + reject() + self.directory_bindings[path] = identity + + def _read(self, name, cap, stable=False): + for path in self.directory_bindings: + self._directory(path, private=path not in (self.cache, self.cache / "logs")) + path = self.root / name + root_fd = None + if os.name == "posix": + # Open every directory relative to its pinned parent. Checking + # lstat followed by a whole-path open alone permits ancestor swaps. + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + root_fd = os.open(self.root.anchor, flags) + try: + current = Path(self.root.anchor) + for component in self.root.parts[1:]: + child = os.open(component, flags, dir_fd=root_fd) + os.close(root_fd) + root_fd = child + current /= component + info = os.fstat(root_fd) + expected = self.directory_bindings.get(current) + if expected is not None and (info.st_dev, info.st_ino) != expected: + reject() + except BaseException: + os.close(root_fd) + raise + try: + return self._read_at(name, path, cap, stable, root_fd) + finally: + if root_fd is not None: + os.close(root_fd) + + def _read_at(self, name, path, cap, stable, root_fd): + before = os.stat(name, dir_fd=root_fd, follow_symlinks=False) if root_fd is not None else path.lstat() + if not stat.S_ISREG(before.st_mode) or before.st_size > cap: + reject() + if os.name == "posix" and (before.st_uid != os.geteuid() or stat.S_IMODE(before.st_mode) != 0o600): + reject() + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(name, flags, dir_fd=root_fd) if root_fd is not None else os.open(path, flags) + try: + opened = os.fstat(fd) + identity = (opened.st_dev, opened.st_ino) + if identity != (before.st_dev, before.st_ino): + reject() + if stable and self.record_identity not in (None, identity): + reject() + if stable: + self.record_identity = identity + with os.fdopen(fd, "rb", closefd=False) as stream: + raw = stream.read(cap + 1) + if len(raw) > cap: + reject() + after = os.stat(name, dir_fd=root_fd, follow_symlinks=False) if root_fd is not None else path.lstat() + if identity != (after.st_dev, after.st_ino): + if stable: + reject() + raise AckPending() + return raw + finally: + os.close(fd) + + def snapshot(self, closed=False): + first = self._read("status.json", STATUS_BYTES) + status = parse(first) + shape(status, STATUS_FIELDS) + if status["writer_revision"] != 1 or type(status["writer_revision"]) is not int or status["sink_instance"] != self.instance: + reject() + for name in ("accepted_records", "written_sequence", "written_records"): + integer(status[name], RECORD_LIMIT) + integer(status["written_bytes"], FILE_BYTES) + if integer(status["first_error_code"], 255) != 0 or type(status["closed"]) is not bool: + reject() + canonical_digest(status["rolling_sha256"]) + if (status["written_sequence"] != status["written_records"] + or status["written_records"] > status["accepted_records"]): + reject() + raw = self._read("records.jsonl", FILE_BYTES, stable=True) + if self._read("status.json", STATUS_BYTES) != first: + raise AckPending() + if (status["written_bytes"] < len(self.last_bytes) + or not raw.startswith(self.last_bytes)): + reject() + acknowledged = raw[:status["written_bytes"]] + if len(acknowledged) != status["written_bytes"] or digest(acknowledged) != status["rolling_sha256"]: + reject() + lines = acknowledged.splitlines(keepends=True) + if len(lines) != status["written_records"]: + reject() + payloads = {} + for number, line in enumerate(lines, 1): + if not line.endswith(b"\n") or len(line) > RECORD_BYTES: + reject() + envelope = parse(line) + shape(envelope, ENVELOPE_FIELDS) + if (type(envelope["writer_revision"]) is not int or envelope["writer_revision"] != 1 + or envelope["sink_instance"] != self.instance + or integer(envelope["record_sequence"], RECORD_LIMIT, True) != number): + reject() + canonical_digest(envelope["payload_sha256"]) + prefix = (f'{{"writer_revision":1,"sink_instance":"{self.instance}",' + f'"record_sequence":{number},"payload":').encode() + suffix = (',"payload_sha256":"' + envelope["payload_sha256"] + '"}\n').encode() + if not line.startswith(prefix) or not line.endswith(suffix): + reject() + if digest(line[len(prefix):-len(suffix)]) != envelope["payload_sha256"]: + reject() + payload = envelope["payload"] + validate_payload(payload) + request = payload["request_id"] + if request not in self.allowed or request in payloads: + reject() + payloads[request] = payload + self.last_bytes = acknowledged + if (len(raw) != status["written_bytes"] or status["accepted_records"] != status["written_records"]): + if status["closed"]: + reject() + raise AckPending() + if closed and not status["closed"]: + raise AckPending() + return status, payloads + + def _wait(self, deadline, closed=False, required_id=None): + until = min(deadline, time.monotonic() + 5) + while time.monotonic() < until: + try: + snapshot = self.snapshot(closed) + if time.monotonic() >= until: + raise TraceRejected("native projection acknowledgement exceeded its original deadline") + if required_id is not None and required_id not in snapshot[1]: + if snapshot[0]["closed"]: + reject() + raise AckPending() + return snapshot + except AckPending: + time.sleep(min(.01, max(0, until - time.monotonic()))) + raise TraceRejected("native projection durable acknowledgement incomplete") + + def collect(self, measured, native, identity, logical_id, deadline): + attempts = receipt_ids(measured.get("resolve_trace_receipt"), logical_id) + if self.allowed.intersection(attempts): + reject() + self.allowed.update(attempts) + final = attempts[-1] + status, payloads = self._wait(deadline, required_id=final) + payload = payloads[final] + bindings = { + "instance_id": native["instance_id"], "root_commit_oid": "sha1:" + identity["global_commit"], + "root_tree_oid": "sha1:" + identity["global_tree"], + "native_certificate_receipt_id": native["certificate_receipt_id"], + "native_writer_epoch": native["writer_epoch"], "native_publication_sequence": native["sequence"], + "namespace_view_id": identity["namespace_view_id"], "scope": "/project", + "snapshot_id": measured["snapshot_id"], + } + if any(type(value) is not type(payload[key]) or payload[key] != value for key, value in bindings.items()): + reject() + if (measured["namespace_view_id"] != payload["namespace_view_id"] + or integer(measured["publication_sequence"], positive=True) != payload["native_publication_sequence"] + or measured.get("descriptor_bytes_hex") != descriptor_bytes(payload).hex()): + reject() + self.finals[final] = payload + return {"instrumentation_mode": "typed-projection-writer-v1", "payload": payload, + "prior_attempt_observations": [payloads[x] for x in attempts[:-1] if x in payloads], + "sink_instance": self.instance, "durable_written_sequence": status["written_sequence"]} + + def finish(self, expected, deadline): + status, payloads = self._wait(deadline, closed=True) + if len(self.finals) != expected or any(payloads.get(key) != value for key, value in self.finals.items()): + reject() + return status diff --git a/tests/mst2-e2e/test_commit_update_bench.py b/tests/mst2-e2e/test_commit_update_bench.py new file mode 100644 index 00000000..0844e83c --- /dev/null +++ b/tests/mst2-e2e/test_commit_update_bench.py @@ -0,0 +1,357 @@ +"""Acceptance checks for the real runner's fences, Git oracle and durability.""" + +import hashlib +from datetime import datetime, timedelta, timezone +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import time +import unittest +from unittest.mock import patch + +SOURCE = Path(__file__).with_name("commit_update_bench.py") +SPEC = importlib.util.spec_from_file_location("real_bench", SOURCE) +BENCH = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BENCH) +CI_SPEC = importlib.util.spec_from_file_location("real_bench_ci", SOURCE.with_name("commit_update_ci.py")) +CI = importlib.util.module_from_spec(CI_SPEC) +CI_SPEC.loader.exec_module(CI) + + +class CommitUpdateBenchTests(unittest.TestCase): + def test_owned_native_bootstrap_rejects_other_database_host_and_noncanonical_instance(self): + database = "mst2_bench_" + "a" * 32 + instance = "6ab219b0-4275-45ba-9d7b-7b0b633018cd" + env = {"PGHOST": "127.0.0.1", "PGDATABASE": database} + cases = [("existing_database", instance, env), + (database, instance, dict(env, PGDATABASE="other")), + (database, instance, dict(env, PGHOST="remote.example")), + (database, instance.upper(), env), + (database, "' OR 1=1 --", env)] + with patch.object(CI.bench, "command") as command: + for db, uuid, connection in cases: + with self.assertRaises(ValueError): + CI.initialize_owned_native(db, uuid, connection, time.monotonic() + 30) + command.assert_not_called() + + def test_owned_native_bootstrap_independent_readback_rejects_stale_root_or_epoch(self): + database = "mst2_bench_" + "a" * 32 + instance = "6ab219b0-4275-45ba-9d7b-7b0b633018cd" + tree = "2" * 40 + raw = b"40000 project\0" + bytes.fromhex(tree) + root_tree = hashlib.sha1(b"tree " + str(len(raw)).encode() + b"\0" + raw).hexdigest() + rows = [{"path": "/", "commit": "3" * 40, "tree": root_tree, "commit_tree": root_tree, + "database": database, "raw_tree": raw.hex()}] + native = {"instance_id": instance, "root_commit": "3" * 40, "root_tree": root_tree, + "writer_epoch": 1, "sequence": 0, "state": "INITIALIZING", "certificate_receipt_id": None} + env = {"PGHOST": "127.0.0.1", "PGDATABASE": database} + with patch.object(CI.bench, "command"), patch.object(CI.bench, "query", side_effect=[rows, native]): + record = CI.initialize_owned_native(database, instance, env, time.monotonic() + 30) + self.assertEqual(record["global_commit"], rows[0]["commit"]) + self.assertEqual(record["global_tree"], root_tree) + self.assertEqual(record["correctness"], "PASS") + # Before the Git route has materialized /project, only the bootstrap + # fence can pass. The normal post-readiness fence remains stricter. + with self.assertRaises(AssertionError): + BENCH.validate_identity(rows, "1" * 40, tree, database) + for wrong in [None, dict(native, root_commit="4" * 40), dict(native, writer_epoch=2), + dict(native, instance_id="different"), dict(native, sequence=1), + dict(native, state="READY"), dict(native, certificate_receipt_id=1)]: + with patch.object(CI.bench, "command"), patch.object(CI.bench, "query", side_effect=[rows, wrong]): + with self.assertRaises(AssertionError): + CI.initialize_owned_native(database, instance, env, time.monotonic() + 30) + + for wrong_rows in [None, [], rows * 2, [dict(rows[0], path="/project")], + [dict(rows[0], database="other")], + [dict(rows[0], commit_tree="4" * 40)], + [dict(rows[0], raw_tree=(raw + b"tampered").hex())]]: + with patch.object(CI.bench, "command"), patch.object(CI.bench, "query", side_effect=[wrong_rows, native]): + with self.assertRaises(AssertionError): + CI.initialize_owned_native(database, instance, env, time.monotonic() + 30) + + def test_failure_records_expose_fixed_phase_without_private_exception_text(self): + with self.assertRaises(BENCH.PhaseFailure) as failed: + with BENCH.phase("owned_native_initialization"): + raise AssertionError("private-token-do-not-log") + record = BENCH.failure_record(failed.exception) + self.assertEqual(record, {"execution_failed": True, "error_type": "AssertionError", + "phase": "owned_native_initialization"}) + self.assertNotIn("private-token", json.dumps(record)) + with self.assertRaises(BENCH.PhaseFailure) as nested: + with BENCH.phase("commit_update_benchmark"): + with BENCH.phase("updated_publication_identity"): + raise KeyError("private-connection-string") + self.assertEqual(BENCH.failure_record(nested.exception)["phase"], "updated_publication_identity") + + def test_command_failure_keeps_only_closed_typed_snapshot_diagnostics(self): + private = b'{"record":"measurement_failure","stage":"metadata","snapshot_error_code":"IntegrityError"}\n' + with self.assertRaises(BENCH.PhaseFailure) as failed: + with BENCH.phase("scorpio_sync"): + raise BENCH.CommandFailure("mst2_update_measure", 1, private) + self.assertEqual(BENCH.failure_record(failed.exception), { + "execution_failed": True, "error_type": "CommandFailure", "phase": "scorpio_sync", + "command": "mst2_update_measure", "exit_status": 1, "measurement_stage": "metadata", + "snapshot_error_code": "IntegrityError", + }) + self.assertNotIn("private-token", json.dumps(BENCH.failure_record(failed.exception))) + for stderr in [b"private-token", b"Error: SnapshotError { code: IntegrityError, message: x }", + b'{"record":"measurement_failure","stage":"metadata","snapshot_error_code":"Unknown"}', + b'{"record":"measurement_failure","stage":"secret-stage"}', + b'{"record":"measurement_failure","stage":"metadata","message":"private-token"}', + private + b"private-token", b"[]", b"null", b"\xff", b" " * 4097]: + error = BENCH.CommandFailure("mst2_update_measure", 1, stderr) + self.assertNotIn("snapshot_error_code", BENCH.failure_record(error)) + self.assertNotIn("measurement_stage", BENCH.failure_record(error)) + # Git/SQL messages cannot impersonate a driver's typed error. + error = BENCH.CommandFailure("git", 128, private) + self.assertEqual(BENCH.failure_record(error), { + "execution_failed": True, "error_type": "CommandFailure", "command": "git", "exit_status": 128, + }) + + def test_query_reports_safe_phase_without_echoing_invalid_output(self): + for sql, phase in [(BENCH.IDENTITY_SQL, "Git identity"), (BENCH.NATIVE_SQL, "native publication")]: + with patch.object(BENCH, "command", return_value=b"private-token-do-not-log"): + with self.assertRaises(AssertionError) as error: + BENCH.query(sql, time.monotonic() + 30) + self.assertIn(phase, str(error.exception)) + self.assertNotIn("private-token", str(error.exception)) + with patch.object(BENCH, "command", return_value=b"null\n"): + self.assertIsNone(BENCH.query(BENCH.NATIVE_SQL, time.monotonic() + 30)) + + def test_workflow_recovery_preserves_deadline_and_rejects_extension_before_setup(self): + workflow = SOURCE.parents[2] / ".github/workflows/mst2-real-update.yml" + script = textwrap.dedent(workflow.read_text().split("python3 - <<'PY'\n", 1)[1].split("\n PY", 1)[0]) + deadline = (datetime.now(timezone.utc) + timedelta(minutes=20)).isoformat() + with tempfile.TemporaryDirectory() as temp: + output = Path(temp) / "github-env" + env = dict(os.environ, GITHUB_ENV=str(output), RUNNER_TEMP=temp, + GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT="2", RESUME_SPEC="", + DEADLINE_INPUT=deadline, TIMEOUT_INPUT="19") + subprocess.run([os.sys.executable, "-c", script], check=True, env=env, capture_output=True) + self.assertIn("MST2_SESSION_DEADLINE=" + deadline, output.read_text()) + output.unlink() + spec = json.dumps({"session_deadline_utc": deadline, "timeout_minutes": 19}) + subprocess.run([os.sys.executable, "-c", script], check=True, + env=dict(env, DEADLINE_INPUT="", TIMEOUT_INPUT="", RESUME_SPEC=spec), capture_output=True) + self.assertIn("MST2_SESSION_DEADLINE=" + deadline, output.read_text()) + output.unlink() + for bad in [dict(env, TIMEOUT_INPUT="21"), + dict(env, DEADLINE_INPUT="2000-01-01T00:00:00Z"), + dict(env, DEADLINE_INPUT="2099-01-01T00:00:00Z"), + dict(env, DEADLINE_INPUT=deadline[:-6]), + dict(env, DEADLINE_INPUT=deadline[:-6] + "+08:00"), + dict(env, DEADLINE_INPUT="", RESUME_SPEC='{"timeout_minutes":19}')]: + result = subprocess.run([os.sys.executable, "-c", script], env=bad, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output.exists()) + + def test_ci_setup_plan_cannot_create_local_resources(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "nonexistent-owned-root" + output = subprocess.check_output([os.sys.executable, str(SOURCE.with_name("commit_update_ci.py")), + "--run-root", str(root)]) + self.assertFalse(json.loads(output)["execute"]) + self.assertFalse(root.exists()) + with patch.dict(os.environ, {"GITHUB_ACTIONS": "false"}): + with self.assertRaises(ValueError): + CI.hosted_root(root) + + def test_ci_compose_has_unique_network_and_rejects_host_bind_storage(self): + services = {name: {"image": "pinned-upstream", "networks": {"default": None}} + for name in ("postgres", "redis", "rustfs", "rustfs-init")} + config = {"services": services, "networks": {"default": {"name": "shared-old-network"}}} + ports = {"postgres": 39001, "redis": 39002, "rustfs": 39003} + with patch.object(CI.bench, "command", return_value=json.dumps(config).encode()): + safe = CI.dependencies(Path("unused"), "owned-project", ports, time.monotonic() + 60) + self.assertEqual(safe["networks"]["default"]["name"], "owned-project-network") + self.assertEqual(safe["services"]["postgres"]["ports"][0]["host_ip"], "127.0.0.1") + services["postgres"]["volumes"] = ["/existing/data:/var/lib/postgresql"] + with patch.object(CI.bench, "command", return_value=json.dumps(config).encode()): + with self.assertRaises(AssertionError): + CI.dependencies(Path("unused"), "owned-project", ports, time.monotonic() + 60) + + def test_ci_cleanup_rejects_replaced_configuration_before_mutation(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / "dependencies.json").write_text("tampered") + (root / "owned.json").write_text(json.dumps({"project": "owned", "compose_sha256": "0" * 64})) + with patch.object(CI.bench, "command") as command: + with self.assertRaises(AssertionError): + CI.stop_owned(root, "owned", time.monotonic() + 60) + command.assert_not_called() + + def test_ci_literal_template_roundtrip_preserves_nested_token_configuration(self): + config = {"base_dir": "/private/job space", "mst2": {"enabled": True, "instance_uuid": "uuid"}, + "git": {"push_tokens": [{"name": "job", "token": "${file:/private/token}", "paths": ["/project"]}]}, + "lfs": {"local": {"lfs_file_path": "${base_dir}/lfs"}}} + self.assertEqual(CI.bench.tomllib.loads(CI.toml(config)), config) + + def test_remote_service_cannot_be_confused_with_loopback_git_target(self): + self.assertEqual(BENCH.endpoint_pair("http://127.0.0.1:39091", "http://127.0.0.1:39091/project"), 39091) + for base, remote in [("http://127.0.0.1:39091", "http://127.0.0.1:39092/project"), + ("http://example.invalid:39091", "http://example.invalid:39091/project"), + ("http://user:secret@127.0.0.1:39091", "http://127.0.0.1:39091/project")]: + with self.assertRaises(ValueError): + BENCH.endpoint_pair(base, remote) + + def test_plan_only_never_reads_service_pid_or_creates_results(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "new-results" + output = subprocess.check_output([ + os.sys.executable, str(SOURCE), "--base-url", "http://127.0.0.1:39091", + "--git-url", "http://127.0.0.1:39091/project", "--database", "benchmark_only", + "--instance-id", "uuid-not-read-in-plan", "--expect-initial-commit", "not-read-in-plan", + "--service-pid", "99999999", "--driver", str(Path(temp) / "absent-driver"), + "--driver-sha256", "not-read-in-plan", + "--run-root", str(root)]) + plan = json.loads(output) + self.assertFalse(plan["execute"]) + self.assertEqual(plan["rounds"], 3) + self.assertEqual(plan["max_wall_seconds"], 14400) + self.assertFalse(root.exists()) + + def test_wrong_binary_hash_fails_before_workload_mutation(self): + from types import SimpleNamespace + with tempfile.TemporaryDirectory() as temp: + driver = Path(temp) / "driver" + driver.write_bytes(b"fixed reviewed binary") + options = SimpleNamespace(driver=driver, driver_sha256=hashlib.sha256(driver.read_bytes()).hexdigest()) + BENCH.driver_binding(options) + driver.write_bytes(b"different binary") + with self.assertRaises(AssertionError): + BENCH.driver_binding(options) + + def test_identity_fails_for_stale_project_and_wrong_global_tree(self): + commit, tree = "1" * 40, "2" * 40 + raw = b"40000 project\0" + bytes.fromhex(tree) + root_tree = hashlib.sha1(b"tree " + str(len(raw)).encode() + b"\0" + raw).hexdigest() + rows = [{"path": "/", "commit": "3" * 40, "tree": root_tree, "commit_tree": root_tree, + "database": "benchmark_only", "raw_tree": raw.hex()}, + {"path": "/project", "commit": commit, "tree": tree, "commit_tree": tree, + "database": "benchmark_only", "raw_tree": None}] + identity = BENCH.validate_identity(rows, commit, tree, "benchmark_only") + self.assertEqual(identity["project_commit"], commit) + with self.assertRaises(AssertionError): + BENCH.validate_identity(rows, "4" * 40, tree, "benchmark_only") + rows[0]["raw_tree"] = (b"40000 unrelated\0" + bytes.fromhex(tree)).hex() + with self.assertRaises(AssertionError): + BENCH.validate_identity(rows, commit, tree, "benchmark_only") + + def test_native_certificate_requires_same_root_and_outbox(self): + identity = {"global_commit": "a", "global_tree": "b", "project_commit": "c", "project_tree": "d"} + native = {"instance_id": "instance", "root_commit": "a", "root_tree": "b", "state": "READY", + "writer_epoch": 1, "certificate_id": 7, "receipt_id": 7, "certificate_namespace": "/", + "certificate_epoch": 1, "receipt_epoch": 1, "writer_kind": "trunk_push", + "request_digest_version": 1, "request_digest": "sha256:" + "0" * 64, + "old_oid": "old-root", "old_root_commit": "old-root", "old_path_commit": "old-path", + "new_oid": "c", "receipt_namespace": "/project", + "certificate_receipt_id": 7, "outbox_id": 9, "certificate_commit": "a", + "certificate_tree": "b", "certificate_instance": "instance", "certificate_sequence": 2, + "sequence": 2, "path_commit": "c", "path_tree": "d", "origin_path": "/project", + "origin_ref": "refs/heads/main", "native_certificate_version": 1, + "outbox_namespace": "/project", "outbox_sequence": 11, "origin_sequence": 11} + BENCH.validate_native(native, identity, "instance", True) + for field, value in [("outbox_id", None), ("certificate_commit", "stale"), ("certificate_sequence", 1), + ("certificate_namespace", "/project"), ("receipt_epoch", 2), + ("writer_epoch", 2), ("sequence", -1), ("writer_kind", "other"), + ("request_digest_version", 0), ("request_digest", "sha256:" + "A" * 64), + ("old_oid", "unrelated"), ("new_oid", "stale"), ("old_path_commit", "c"), + ("receipt_namespace", "/unrelated")]: + wrong = dict(native, **{field: value}) + with self.assertRaises(AssertionError): + BENCH.validate_native(wrong, identity, "instance", True) + + def test_real_git_changes_one_body_then_renames_without_new_body(self): + with tempfile.TemporaryDirectory() as temp: + repo = Path(temp) / "fixture" + deadline = time.monotonic() + 120 + subprocess.run(["git", "init", "-q", "-b", "main", str(repo)], check=True) + env = BENCH.clean_env({"GIT_AUTHOR_NAME": "test", "GIT_COMMITTER_NAME": "test", + "GIT_AUTHOR_EMAIL": "test@example.invalid", "GIT_COMMITTER_EMAIL": "test@example.invalid"}) + BENCH.git(repo, deadline, "commit", "--allow-empty", "-qm", "seed", env=env) + manifests = [] + for version in ("v1", "v2", "v3"): + commit, _ = BENCH.create_version(repo, 1, version, True, deadline) + manifests.append(BENCH.expected_manifest(repo, commit, deadline)) + sets = [{f["content_digest"] for f in manifest["files"]} for manifest in manifests] + self.assertEqual(len(sets[1] - sets[0]), 1) + self.assertEqual(sets[2], sets[1]) + self.assertEqual(len(manifests[0]["files"]), len(manifests[2]["files"])) + self.assertTrue(any(f["rel_path"].startswith("r01/renamed-m001/") for f in manifests[2]["files"])) + self.assertIn("empty-a", manifests[2]["directories"]) + self.assertIn("empty-b", manifests[2]["directories"]) + self.assertIn("alias-r01-m007", manifests[2]["directories"]) + # A second repetition needs a distinct cold content set and tree. + commit, _ = BENCH.create_version(repo, 2, "v1", True, deadline) + new = BENCH.expected_manifest(repo, commit, deadline) + self.assertTrue(all(f["rel_path"].startswith(("r02/", "alias-r02-m007/")) for f in new["files"])) + self.assertFalse(sets[0] & {f["content_digest"] for f in new["files"]}) + + def test_independent_git_worktree_oracle_rejects_tampered_bytes_and_extra_entries(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path = root / "file" + path.write_bytes(b"original") + # Windows has no Unix execute-bit contract, so derive the local + # kind for this byte-integrity test independently of Git checkout. + kind = "executable" if path.stat().st_mode & 0o111 else "regular" + expected = {"files": [{"rel_path": "file", "fs_kind": kind, "size": 8, + "content_digest": "sha256:" + hashlib.sha256(b"original").hexdigest()}]} + BENCH.verify_worktree(root, expected) + path.write_bytes(b"tampered") + with self.assertRaises(AssertionError): + BENCH.verify_worktree(root, expected) + path.write_bytes(b"original") + (root / "extra").write_bytes(b"extra") + with self.assertRaises(AssertionError): + BENCH.verify_worktree(root, expected) + + def test_timeout_terminates_whole_child_group_then_kills_if_term_is_ignored(self): + class Hung: + pid = 12345 + returncode = None + calls = 0 + def communicate(self, data=None, timeout=None): + self.calls += 1 + if self.calls <= 2: + raise subprocess.TimeoutExpired("ignored", timeout) + return b"", b"" + child = Hung() + with patch.object(BENCH.budget_module.subprocess, "Popen", return_value=child), \ + patch.object(BENCH.budget_module, "PinnedProcess", return_value=child), \ + patch.object(BENCH.budget_module, "process_start", return_value="1"), \ + patch.object(BENCH.budget_module, "group_members", return_value=[12345]), \ + patch.object(BENCH.budget_module, "stop_group") as stop_group, \ + patch.object(BENCH.os, "killpg", create=True) as killpg, \ + patch.object(BENCH.budget_module.signal, "SIGKILL", 9, create=True): + with self.assertRaises(TimeoutError): + BENCH.command(["driver"], time.monotonic() + 30) + self.assertEqual([call.args[1] for call in killpg.call_args_list], + [BENCH.budget_module.signal.SIGTERM, BENCH.budget_module.signal.SIGKILL]) + self.assertTrue(all(call.args[0] == 12345 for call in killpg.call_args_list)) + self.assertEqual(child.calls, 3) + self.assertEqual(stop_group.call_args.args[:2], (12345, "1")) + self.assertIs(stop_group.call_args.args[3], child) + + @unittest.skipUnless(hasattr(os, "O_DIRECTORY") and hasattr(os, "O_NOFOLLOW"), "Linux durability primitive") + def test_durable_flush_includes_regular_files_and_directories_without_following_symlinks(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "worktree" + root.mkdir() + (root / "nested").mkdir() + (root / "nested/data").write_bytes(b"fixed commit") + (root / "link").symlink_to("nested/data") + result = BENCH.fsync_tree(root, time.monotonic() + 30) + self.assertEqual(result, {"regular_files": 1, "directories": 2, "file_bytes": 12}) + alias = Path(temp) / "alias" + alias.symlink_to(root, target_is_directory=True) + with self.assertRaises(AssertionError): + BENCH.fsync_tree(alias, time.monotonic() + 30) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mst2-e2e/test_commit_update_budget.py b/tests/mst2-e2e/test_commit_update_budget.py new file mode 100644 index 00000000..5935d9fd --- /dev/null +++ b/tests/mst2-e2e/test_commit_update_budget.py @@ -0,0 +1,544 @@ +"""Budget admissions and actual Linux process cleanup, without a deployment.""" + +from datetime import datetime, timedelta, timezone +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import commit_update_bench as bench +import commit_update_budget as budget +import commit_update_ci as ci + + +class SessionBudgetTests(unittest.TestCase): + def test_communicate_wait_keeps_leader_pinned_until_explicit_group_cleanup(self): + process = object.__new__(budget.PinnedProcess) + process._child_created = False + process.reap_pinned = True + process.returncode = None + process.pid = 123 + process.args = ["owned"] + observation = SimpleNamespace(si_code=1, si_status=0) + with patch.multiple(budget.os, P_PID=1, WEXITED=2, WNOHANG=4, WNOWAIT=8, + CLD_EXITED=1, create=True), \ + patch.object(budget.os, "waitid", return_value=observation, create=True) as observe, \ + patch.object(subprocess.Popen, "wait", return_value=0) as reap: + self.assertEqual(process.wait(timeout=1), 0) + self.assertEqual(process._wait(timeout=1), 0) + self.assertEqual(process.wait(timeout=1), 0) + self.assertIsNone(process.returncode) + reap.assert_not_called() + observe.assert_called_with(1, 123, 2 | 4 | 8) + budget.reap_owned(process, time.monotonic() + 1) + reap.assert_called_once() + self.assertFalse(process.reap_pinned) + + def test_timeout_never_kills_reaped_or_replaced_leader_group(self): + for changed in ("reaped", "replaced"): + class Child: + pid = 123 + returncode = None + def communicate(self, *args, **kwargs): + if not hasattr(self, "term_sent"): + self.term_sent = True + raise subprocess.TimeoutExpired("owned", 1) + if changed == "reaped": + self.returncode = 0 + return b"", b"" + child = Child() + scans = iter(([123], AssertionError("owned group leader identity was replaced"))) + def members(*_): + result = next(scans) + if isinstance(result, Exception): + raise result + return result + with patch.object(budget.sys, "platform", "linux"), \ + patch.object(budget.signal, "SIGKILL", 9, create=True), \ + patch.object(budget, "PinnedProcess", return_value=child), \ + patch.object(budget, "process_start", return_value="1"), \ + patch.object(budget, "group_members", side_effect=members), \ + patch.object(budget, "reap_exited"), \ + patch.object(budget.os, "killpg", create=True) as send: + with self.assertRaises(AssertionError): + budget.run_process(["owned"], time.monotonic() + 30) + send.assert_called_once_with(123, budget.signal.SIGTERM) + + def test_open_pipe_timeout_reaps_exited_leader_without_extending_wait_or_masking_failure(self): + class Child: + pid = 123 + returncode = None + stdin = None + stdout = io.BytesIO() + stderr = io.BytesIO() + def communicate(self, *args, **kwargs): + raise subprocess.TimeoutExpired("owned", 1) + def release_reap(self): + self.released = True + def wait(self, timeout=None): + self.wait_timeout = timeout + self.returncode = 0 + child = Child() + with patch.object(budget.sys, "platform", "linux"), \ + patch.object(budget, "PinnedProcess", return_value=child), \ + patch.object(budget, "process_start", return_value="1"), \ + patch.object(budget, "group_members", return_value=[]), \ + patch.object(budget.signal, "SIGKILL", 9, create=True), \ + patch.multiple(budget.os, P_PID=1, WEXITED=2, WNOHANG=4, WNOWAIT=8, create=True), \ + patch.object(budget.os, "waitid", return_value=SimpleNamespace(si_status=0), create=True), \ + patch.object(budget.os, "killpg", create=True) as send: + with self.assertRaisesRegex(TimeoutError, "owned child did not reap"): + budget.run_process(["owned"], time.monotonic() + 30) + send.assert_not_called() + self.assertTrue(child.released) + self.assertEqual(child.wait_timeout, 0) + self.assertEqual(child.returncode, 0) + + def test_group_timeout_reaps_exited_direct_leader_and_preserves_timeout(self): + child = SimpleNamespace(pid=123, returncode=None) + waits = [] + def wait(timeout=None): + waits.append(timeout) + child.returncode = 0 + child.wait = wait + with patch.object(budget.sys, "platform", "linux"), \ + patch.object(budget, "group_members", return_value=[456]), \ + patch.object(budget.signal, "SIGKILL", 9, create=True), \ + patch.object(budget.os, "killpg", create=True), \ + patch.multiple(budget.os, P_PID=1, WEXITED=2, WNOHANG=4, WNOWAIT=8, create=True), \ + patch.object(budget.os, "waitid", return_value=SimpleNamespace(si_status=0), create=True): + with self.assertRaisesRegex(TimeoutError, "owned group remained active"): + budget.stop_group(123, "1", time.monotonic() - 1, child) + self.assertEqual(waits, [0]) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux escaped open-pipe timeout") + def test_exited_leader_with_escaped_pipe_holder_is_reaped_without_signalling_escape(self): + import signal + with tempfile.TemporaryDirectory() as temp: + pids = Path(temp) / "pids" + script = textwrap.dedent("""\ + import os, signal, sys, time + leader = os.getpid() + child = os.fork() + if child == 0: + os.setsid() + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with open(sys.argv[1], 'w') as stream: + stream.write(str(leader) + ' ' + str(os.getpid())) + while True: time.sleep(.01) + while not os.path.exists(sys.argv[1]): time.sleep(.01) + """) + deadline = time.monotonic() + 2 + try: + with patch.object(budget.os, "killpg", wraps=os.killpg) as send: + with self.assertRaises(TimeoutError): + budget.run_process([sys.executable, "-c", script, str(pids)], deadline) + leader, child = map(int, pids.read_text().split()) + self.assertFalse(Path(f"/proc/{leader}").exists()) + self.assertTrue(Path(f"/proc/{child}").exists()) + self.assertTrue(all(call.args[0] != child for call in send.call_args_list)) + self.assertLess(time.monotonic(), deadline + .15) + finally: + if pids.exists(): + _, child = map(int, pids.read_text().split()) + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + + @unittest.skipUnless(sys.platform == "linux", "actual Linux communicate KeyboardInterrupt") + def test_keyboard_interrupt_keeps_leader_pinned_until_same_group_descendant_cleanup(self): + import signal + import threading + with tempfile.TemporaryDirectory() as temp: + pids = Path(temp) / "pids" + script = textwrap.dedent("""\ + import os, signal, sys, time + leader = os.getpid() + child = os.fork() + if child == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with open(sys.argv[1], 'w') as stream: + stream.write(str(leader) + ' ' + str(os.getpid())) + while True: time.sleep(.01) + while not os.path.exists(sys.argv[1]): time.sleep(.01) + """) + original = budget.stop_group + observed = [] + def stop(pgid, started, deadline, process=None): + self.assertIsNone(process.returncode) + self.assertTrue(Path(f"/proc/{pgid}").exists()) + observed.append(pgid) + return original(pgid, started, deadline, process) + timer = threading.Timer(.3, os.kill, args=(os.getpid(), signal.SIGINT)) + deadline = time.monotonic() + 3 + try: + timer.start() + with patch.object(budget, "stop_group", side_effect=stop): + with self.assertRaises(KeyboardInterrupt): + budget.run_process([sys.executable, "-c", script, str(pids)], deadline) + leader, child = map(int, pids.read_text().split()) + self.assertEqual(observed, [leader]) + self.assertFalse(Path(f"/proc/{leader}").exists()) + proc = Path(f"/proc/{child}/stat") + self.assertTrue(not proc.exists() or proc.read_text().rsplit(") ", 1)[1].split()[0] == "Z") + self.assertLess(time.monotonic(), deadline) + finally: + timer.cancel() + timer.join() + if pids.exists(): + _, child = map(int, pids.read_text().split()) + try: + os.kill(child, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_build_setup_round_admission_uses_one_anchor_and_never_grants_more_time(self): + with patch.object(budget.time, "time", return_value=0), \ + patch.object(budget.time, "monotonic", return_value=0): + shared = budget.SessionBudget("1970-01-01T03:55:00Z", 3) + self.assertEqual(shared.cleanup_deadline, 220 * 60) + self.assertEqual(shared.measurement_deadline, 200 * 60) + for now, stage, expected in [(10, "server-build", 45), (45, "client-build", 65), + (65, "fences", 75), (75, "setup", 85)]: + with patch.object(budget.time, "monotonic", return_value=now * 60): + self.assertEqual(shared.stage_deadline(stage), expected * 60) + for number, now in [(1, 85), (2, 110), (3, 135)]: + with patch.object(budget.time, "monotonic", return_value=now * 60): + self.assertEqual(shared.round_deadline(number), (now + 25) * 60) + with patch.object(budget.time, "monotonic", return_value=160 * 60): + self.assertEqual(shared.report_deadline(), 170 * 60) + for stage in budget.STAGES: + with patch.object(budget.time, "monotonic", return_value=219 * 60): + with self.assertRaises(TimeoutError): + shared.stage_deadline(stage) + for number in (1, 2, 3): + with patch.object(budget.time, "monotonic", return_value=219 * 60): + with self.assertRaises(TimeoutError): + shared.round_deadline(number) + self.assertEqual(shared.cleanup_deadline, 220 * 60) + self.assertIs(budget.from_options(SimpleNamespace(budget=shared)), shared) + + def test_short_recovery_rejects_medium_before_build_or_owned_resource_mutation(self): + deadline = (datetime.now(timezone.utc) + timedelta(minutes=30)).isoformat() + shared = budget.SessionBudget(deadline, 3) + with patch.object(budget.subprocess, "Popen") as start: + with self.assertRaises(TimeoutError): + budget.run_process(["unused"], shared.stage_deadline("server-build")) + start.assert_not_called() + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "unused" + opts = SimpleNamespace(run_root=root, mega_sha="1" * 40, budget=shared) + with patch.object(ci, "hosted_root", return_value=(root, "owned")), \ + patch.object(ci, "dependencies") as deps: + with self.assertRaises(TimeoutError): + ci.execute(opts) + deps.assert_not_called() + self.assertFalse(root.exists()) + + def test_verified_end_point_rejects_tampering_and_contains_immediate_full_byte_oracle(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + body = b"fixed bytes" + target = root / "file" + target.write_bytes(body) + kind = "executable" if target.stat().st_mode & 0o111 else "regular" + expected = {"files": [{"rel_path": "file", "fs_kind": kind, "size": len(body), + "content_digest": "sha256:" + hashlib.sha256(body).hexdigest()}]} + order = [] + def operation(): + order.append("durable") + return {"durable_complete_ms": 1} + def oracle(_): + order.append("oracle") + bench.verify_worktree(root, expected, time.monotonic() + 30) + with patch.object(bench.time, "monotonic", side_effect=[1, 2, 3, 4, 5, 6, 7]): + measured = bench.durable_verified(operation, oracle) + self.assertEqual(order, ["durable", "oracle"]) + self.assertEqual(measured["durable_byte_oracle_ms"], 4000) + self.assertEqual(measured["durable_verified_ms"], 6000) + target.write_bytes(b"wrong bytes") + with self.assertRaises(AssertionError): + bench.durable_verified(operation, oracle) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux process-group ownership") + def test_actual_capture_and_inherited_io_keep_exited_leader_pinned_through_group_cleanup(self): + original_stop = budget.stop_group + for capture in (True, False): + seen = [] + def stop(pgid, started, deadline, process=None): + self.assertIsNone(process.returncode) + stat = Path(f"/proc/{pgid}/stat").read_text().rsplit(") ", 1)[1].split() + self.assertEqual(stat[0], "Z") + self.assertEqual(stat[19], started) + seen.append(pgid) + return original_stop(pgid, started, deadline, process) + with patch.object(budget, "stop_group", side_effect=stop): + result = budget.run_process([sys.executable, "-c", "pass"], + time.monotonic() + 5, capture=capture) + self.assertEqual(result, (0, b"", b"") if capture else (0, None, None)) + self.assertEqual(len(seen), 1) + self.assertFalse(Path(f"/proc/{seen[0]}").exists()) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux process-group ownership") + def test_actual_timeout_kills_ignoring_descendant_and_reaps_within_original_deadline(self): + with tempfile.TemporaryDirectory() as temp: + pid_file = Path(temp) / "pids" + script = textwrap.dedent("""\ + import os, signal, sys, time + signal.signal(signal.SIGTERM, signal.SIG_IGN) + child = os.fork() + if child == 0: + while True: time.sleep(.05) + with open(sys.argv[1], 'w') as stream: + stream.write(str(os.getpid()) + ' ' + str(child)) + while True: time.sleep(.05) + """) + deadline = time.monotonic() + 2 + with self.assertRaises(TimeoutError): + bench.command([sys.executable, "-c", script, str(pid_file)], deadline) + self.assertLess(time.monotonic(), deadline + .15) + pids = [int(value) for value in pid_file.read_text().split()] + for pid in pids: + proc = Path(f"/proc/{pid}/stat") + self.assertTrue(not proc.exists() or proc.read_text().rsplit(") ", 1)[1].split()[0] == "Z", + "no running child survives the actual timeout") + + @unittest.skipUnless(sys.platform == "linux", "actual Linux owned service finally") + def test_actual_failed_owned_setup_finally_terminates_service_and_uses_fixed_cleanup_deadline(self): + self.actual_owned_failure("setup") + + @unittest.skipUnless(sys.platform == "linux", "actual Linux owned round timeout finally") + def test_actual_round_child_timeout_runs_owned_finally_without_renewing_cleanup(self): + self.actual_owned_failure("round") + + @unittest.skipUnless(sys.platform == "linux", "actual Linux exited leader cleanup") + def test_owned_reaped_leader_cannot_hide_live_descendant_that_ignores_term(self): + self.actual_owned_failure("leader") + + @unittest.skipUnless(sys.platform == "linux", "actual Linux startup identity failure") + def test_failed_owned_startup_identity_still_reaps_before_owned_finally(self): + self.actual_owned_failure("startup") + + @unittest.skipUnless(sys.platform == "linux", "actual Linux startup abort ownership") + def test_command_startup_identity_errors_reap_the_direct_child_without_renewing_deadline(self): + original = budget.PinnedProcess + for error in (PermissionError("fixture identity access"), FileNotFoundError("fixture identity missing")): + spawned = [] + def start(*args, **kwargs): + process = original(*args, **kwargs) + spawned.append(process) + return process + deadline = time.monotonic() + 2 + with patch.object(budget, "PinnedProcess", side_effect=start), \ + patch.object(budget, "process_start", side_effect=error): + with self.assertRaises(PermissionError if isinstance(error, PermissionError) else AssertionError): + budget.run_process([sys.executable, "-c", "import time; time.sleep(30)"], deadline) + self.assertIsNotNone(spawned[0].returncode) + self.assertFalse(Path(f"/proc/{spawned[0].pid}").exists()) + self.assertLess(time.monotonic(), deadline) + for pipe in (spawned[0].stdout, spawned[0].stderr): + pipe.close() + + @unittest.skipUnless(sys.platform == "linux", "actual Linux short child reap") + def test_missing_exited_startup_leader_is_reaped_without_signalling_an_unbound_group(self): + original = budget.PinnedProcess + spawned = [] + def start(*args, **kwargs): + process = original(*args, **kwargs) + # EOF proves the short child exited, without poll/wait releasing its + # PID before the production startup error path reaps it. + self.assertEqual(process.stdout.read(), b"") + spawned.append(process) + return process + deadline = time.monotonic() + 2 + with patch.object(budget, "PinnedProcess", side_effect=start), \ + patch.object(budget, "process_start", side_effect=FileNotFoundError), \ + patch.object(budget.os, "getpgid", side_effect=ProcessLookupError), \ + patch.object(budget.os, "killpg") as signal_group: + self.assertEqual(budget.run_process([sys.executable, "-c", "pass"], deadline), (0, b"", b"")) + signal_group.assert_not_called() + self.assertEqual(spawned[0].returncode, 0) + self.assertFalse(Path(f"/proc/{spawned[0].pid}").exists()) + self.assertLess(time.monotonic(), deadline) + + @unittest.skipUnless(sys.platform == "linux", "actual Linux group scan checkpoint") + def test_empty_group_scan_finishing_after_deadline_cannot_report_cleanup_success(self): + original = budget.group_members + deadline = time.monotonic() + .05 + def late_scan(*args): + members = original(*args) + time.sleep(.07) + return members + with patch.object(budget, "group_members", side_effect=late_scan), \ + patch.object(budget.os, "killpg") as signal_group: + with self.assertRaises(TimeoutError): + budget.stop_group(-1, "0", deadline) + signal_group.assert_not_called() + + @unittest.skipUnless(sys.platform == "linux", "actual Linux successful leader cleanup") + def test_successful_short_command_still_cleans_child_that_closed_inherited_pipes(self): + with tempfile.TemporaryDirectory() as temp: + pid_file = Path(temp) / "child" + script = textwrap.dedent("""\ + import os, signal, sys, time + child = os.fork() + if child == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + for fd in (0, 1, 2): os.close(fd) + with open(sys.argv[1], 'w') as stream: stream.write(str(os.getpid())) + while True: time.sleep(.05) + while not os.path.exists(sys.argv[1]): time.sleep(.01) + """) + deadline = time.monotonic() + 8 + self.assertEqual(bench.command([sys.executable, "-c", script, str(pid_file)], deadline), b"") + child = Path(f"/proc/{int(pid_file.read_text())}/stat") + self.assertTrue(not child.exists() or child.read_text().rsplit(") ", 1)[1].split()[0] == "Z") + self.assertLess(time.monotonic(), deadline) + + def actual_owned_failure(self, failure): + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) / "source" + (source / "target").mkdir(parents=True) + (source / "config").mkdir() + binary = source / "target" / "owned-service" + program = b"#include \nint main(void) { for (;;) pause(); }\n" + if failure == "leader": + program = textwrap.dedent("""\ + #include + #include + #include + #include + int main(int argc, char **argv) { + int sync[2]; char byte; pipe(sync); + if (fork() == 0) { + signal(SIGTERM, SIG_IGN); + close(0); close(1); close(2); + char path[8192]; snprintf(path, sizeof(path), "%s.child", argv[2]); + FILE *file = fopen(path, "w"); fprintf(file, "%d", getpid()); fclose(file); + write(sync[1], "x", 1); for (;;) pause(); + } + read(sync[0], &byte, 1); return 0; + } + """).encode() + subprocess.run(["cc", "-x", "c", "-o", str(binary), "-"], input=program, + check=True, capture_output=True) + template = {"base_dir": "unused", "log": {}, "database": {}, "redis": {}, + "monorepo": {}, "pack": {}, "object_storage": {"s3": {}}, "git": {}} + (source / "config/config-storage-only.toml").write_text(ci.toml(template)) + driver = source / "driver" + driver.write_bytes(b"fixed") + root = Path(temp) / "owned" + shared = budget.SessionBudget((datetime.now(timezone.utc) + timedelta(hours=3)).isoformat(), 3) + shared.cleanup_deadline = time.monotonic() + 8 + setup_deadline = time.monotonic() + (1 if failure == "setup" else 5) + opts = SimpleNamespace(run_root=root, mega_sha="1" * 40, mega_source=source, + mega_binary=binary, driver=driver, + driver_sha256=hashlib.sha256(b"fixed").hexdigest(), budget=shared, + session_deadline_utc=shared.deadline_utc, profile="medium", rounds=3) + cleanup_deadlines = [] + service_pids = [] + original_stop = ci.stop_owned + original_command = bench.command + def stop(*args): + cleanup_deadlines.append(args[2]) + service_pids.append(args[3].pid) + return original_stop(*args) + def command(args, deadline, **_): + self.assertLessEqual(deadline, shared.cleanup_deadline) + if "ls-remote" in args: + return b"2" * 40 + b"\trefs/heads/main\n" + return b"" + class Ready: + status = 200 + def __enter__(self): + return self + def __exit__(self, *_): + return False + def round_timeout(args): + self.assertIs(args.budget, shared) + deadline = min(time.monotonic() + 2, args.budget.cleanup_deadline - 1) + original_command([sys.executable, "-c", "import time; time.sleep(30)"], deadline) + ready = {"return_value": Ready()} if failure == "round" else {"side_effect": OSError} + original_process_start = budget.process_start + def process_start(pid): + if failure == "startup": + raise PermissionError("fixture startup identity failure") + return original_process_start(pid) + # Docker/DB are fixture stubs; the service Popen, /proc identity, + # owned finally, TERM/group cleanup and reap are actual Linux code. + with patch.dict(os.environ), \ + patch.object(ci, "hosted_root", return_value=(root, "owned")), \ + patch.object(shared, "stage_deadline", return_value=setup_deadline), \ + patch.object(shared, "require"), \ + patch.object(ci.bench, "git", side_effect=[b"1" * 40, b""]), \ + patch.object(ci.bench, "command", side_effect=command), \ + patch.object(ci, "dependencies", return_value={"services": {}}), \ + patch.object(ci, "initialize_owned_native", return_value={"correctness": "PASS"}), \ + patch.object(ci, "urlopen", **ready), \ + patch.object(ci.bench, "execute", side_effect=round_timeout), \ + patch.object(budget, "process_start", side_effect=process_start), \ + patch.object(ci, "stop_owned", side_effect=stop), \ + patch("sys.stdout", new_callable=io.StringIO): + expected_error = (PermissionError if failure == "startup" + else bench.PhaseFailure if failure == "round" else RuntimeError) + with self.assertRaises(expected_error): + ci.execute(opts) + self.assertEqual(cleanup_deadlines, [shared.cleanup_deadline]) + state = json.loads((root / "owned.json").read_text()) + self.assertNotIn("service", state) + self.assertFalse(Path(f"/proc/{service_pids[0]}").exists()) + if failure == "leader": + child = Path(f"/proc/{int((root / 'service.toml.child').read_text())}/stat") + self.assertTrue(not child.exists() or child.read_text().rsplit(") ", 1)[1].split()[0] == "Z") + self.assertLess(time.monotonic(), shared.cleanup_deadline) + + def test_cleanup_inventory_failure_keeps_owned_state_and_never_renews_deadline(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + compose = b"{}" + (root / "dependencies.json").write_bytes(compose) + state = {"project": "owned", "compose_sha256": hashlib.sha256(compose).hexdigest()} + (root / "owned.json").write_text(json.dumps(state)) + deadline = time.monotonic() + 1 + seen = [] + def command(args, supplied, **_): + seen.append(supplied) + return b"leftover-owned-container" if "ps" in args else b"" + with patch.object(ci.bench, "command", side_effect=command), \ + patch("sys.stdout", new_callable=io.StringIO) as output: + with self.assertRaises(AssertionError): + ci.stop_owned(root, "owned", deadline) + self.assertNotIn('"correctness": "PASS"', output.getvalue()) + self.assertEqual(seen, [deadline, deadline, deadline]) + self.assertEqual(json.loads((root / "owned.json").read_text()), state) + + def test_cleanup_final_inventory_and_metadata_checkpoints_never_emit_late_pass(self): + for checkpoint in ("inventory", "metadata"): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + compose = b"{}" + (root / "dependencies.json").write_bytes(compose) + state = {"project": "owned", "compose_sha256": hashlib.sha256(compose).hexdigest()} + (root / "owned.json").write_text(json.dumps(state)) + # Inventory and state serialization each have a final checkpoint. + times = [0, 2] if checkpoint == "metadata" else [2] + with patch.object(ci.bench, "command", return_value=b""), \ + patch.object(ci.time, "monotonic", side_effect=times), \ + patch("sys.stdout", new_callable=io.StringIO) as output: + with self.assertRaises(TimeoutError): + ci.stop_owned(root, "owned", 1) + self.assertNotIn('"correctness": "PASS"', output.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mst2-e2e/test_commit_update_projection.py b/tests/mst2-e2e/test_commit_update_projection.py new file mode 100644 index 00000000..3595dc37 --- /dev/null +++ b/tests/mst2-e2e/test_commit_update_projection.py @@ -0,0 +1,309 @@ +"""Actual bounded trace files, retry bindings, acknowledgements and closure.""" + +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import tempfile +import shutil +import subprocess +import threading +import time +import unittest +from unittest.mock import patch +import uuid + +import commit_update_projection as projection +import commit_update_ci as ci +import commit_update_budget as budget + + +class ProjectionCollectionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.cache = Path(self.temp.name) / "cache" + self.cache.mkdir(mode=0o700) + self.logs = self.cache / "logs" + self.logs.mkdir() + self.parent = self.logs / "mst2-native-projection" + self.parent.mkdir(mode=0o700) + self.sink = str(uuid.uuid4()) + self.root = self.parent / self.sink + self.root.mkdir(mode=0o700) + self.instance = str(uuid.uuid4()) + self.logical = "mst2:test:r1:v1:resolve" + self.payload = dict(projection.FIXED) + self.payload.update({key: 0 for key in projection.WORK_FIELDS}) + self.payload.update({ + "request_id": self.logical + ":a1", "instance_id": self.instance, + "root_commit_oid": "sha1:" + "a" * 40, "root_tree_oid": "sha1:" + "b" * 40, + "native_certificate_receipt_id": 11, "native_writer_epoch": 1, + "native_publication_sequence": 3, "scope": "/project", + "namespace_view_id": projection.digest(b"mega.mst2.namespaceview\0" + b"a" * 40), + "metadata_root": "sha256:" + "c" * 64, "projection_elapsed_micros": 42, + }) + self.payload["snapshot_id"] = projection.digest(b"mega.mst2.descriptor\0" + projection.descriptor_bytes(self.payload)) + self.native = {"instance_id": self.instance, "certificate_receipt_id": 11, + "writer_epoch": 1, "sequence": 3} + self.identity = {"global_commit": "a" * 40, "global_tree": "b" * 40, + "namespace_view_id": self.payload["namespace_view_id"]} + self.measured = { + "resolve_trace_receipt": {"logical_request_id": self.logical, + "attempt_ids": [self.logical + ":a1"], + "final_attempt_id": self.logical + ":a1", "retry_count": 0}, + "snapshot_id": self.payload["snapshot_id"], "namespace_view_id": self.payload["namespace_view_id"], + "publication_sequence": 3, "descriptor_bytes_hex": projection.descriptor_bytes(self.payload).hex(), + } + + def line(self, payload, sequence): + body = json.dumps(payload, separators=(",", ":")).encode() + prefix = (f'{{"writer_revision":1,"sink_instance":"{self.sink}",' + f'"record_sequence":{sequence},"payload":').encode() + return prefix + body + (',"payload_sha256":"' + projection.digest(body) + '"}\n').encode() + + def write(self, payloads, closed=False, raw=None, **status_changes): + raw = raw if raw is not None else b"".join(self.line(p, n) for n, p in enumerate(payloads, 1)) + record = self.root / "records.jsonl" + # Keep the actual records inode throughout live appends. + with record.open("wb") as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + record.chmod(0o600) + status = {"writer_revision": 1, "sink_instance": self.sink, "accepted_records": len(payloads), + "written_sequence": len(payloads), "written_records": len(payloads), + "written_bytes": len(raw), "rolling_sha256": projection.digest(raw), + "first_error_code": 0, "closed": closed} + status.update(status_changes) + temporary = self.root / "fixture-status.tmp" + with temporary.open("wb") as stream: + stream.write(json.dumps(status).encode()) + stream.flush() + os.fsync(stream.fileno()) + temporary.chmod(0o600) + os.replace(temporary, self.root / "status.json") + if os.name == "posix": + fd = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def collect(self, collector): + return collector.collect(self.measured, self.native, self.identity, self.logical, time.monotonic() + 1) + + def test_live_durable_ack_then_real_closed_status_is_required(self): + self.write([self.payload]) + collector = projection.ProjectionCollector(self.cache) + result = self.collect(collector) + self.assertEqual(result["payload"], self.payload) + self.assertEqual(result["durable_written_sequence"], 1) + with self.assertRaises(projection.TraceRejected): + collector.finish(1, time.monotonic() + .02) + self.write([self.payload], closed=True) + self.assertTrue(collector.finish(1, time.monotonic() + 1)["closed"]) + + def test_previous_success_with_lost_response_is_only_prior_attempt_diagnostic(self): + first = dict(self.payload) + first["root_commit_oid"] = "sha1:" + "d" * 40 + first["namespace_view_id"] = projection.digest(b"mega.mst2.namespaceview\0" + b"d" * 40) + first["snapshot_id"] = projection.digest(b"mega.mst2.descriptor\0" + projection.descriptor_bytes(first)) + final = dict(self.payload, request_id=self.logical + ":a2") + self.measured["resolve_trace_receipt"].update(attempt_ids=[self.logical + ":a1", self.logical + ":a2"], + final_attempt_id=self.logical + ":a2", retry_count=1) + self.write([first, final], closed=True) + collector = projection.ProjectionCollector(self.cache) + result = self.collect(collector) + self.assertEqual(result["payload"], final) + self.assertEqual(result["prior_attempt_observations"], [first]) + collector.finish(1, time.monotonic() + 1) + + def test_waits_for_real_delayed_durable_record_within_original_deadline(self): + self.write([]) + collector = projection.ProjectionCollector(self.cache) + def acknowledge(): + time.sleep(.03) + self.write([self.payload]) + worker = threading.Thread(target=acknowledge) + worker.start() + try: + self.assertEqual(self.collect(collector)["payload"], self.payload) + finally: + worker.join(timeout=1) + + def test_sticky_errors_closed_gaps_and_bad_status_are_terminal(self): + for changes in ({"first_error_code": 1}, {"first_error_code": True}, + {"written_sequence": 2}, {"accepted_records": 65}, + {"written_bytes": 0}, {"rolling_sha256": "sha256:" + "0" * 64}, + {"closed": 1}, {"unknown": 0}, {"accepted_records": 2, "closed": True}): + with self.subTest(changes=changes): + self.write([self.payload], **changes) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + + def test_closed_shape_payload_types_profiles_and_binding_are_checked(self): + variants = [dict(self.payload, credentials="secret"), dict(self.payload, tree_fetches=True), + dict(self.payload, raw_bytes_hashed=1 << 64), dict(self.payload, schema_version=True), + dict(self.payload, root_tree_oid="sha256:" + "b" * 64), + dict(self.payload, native_certificate_receipt_id=0), + dict(self.payload, scope="/project/"), dict(self.payload, snapshot_id="sha256:" + "d" * 64), + dict(self.payload, native_publication_sequence=4)] + for value in variants: + with self.subTest(value=value): + self.write([value]) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.write([self.payload]) + self.measured["descriptor_bytes_hex"] = "00" + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + + def test_raw_payload_hash_duplicate_keys_truncation_and_duplicate_request_fail(self): + line = self.line(self.payload, 1) + bad = [line.replace(b'"tree_fetches":0', b'"tree_fetches":1'), + line.replace(b'"tree_fetches":0', b'"tree_fetches":0,"tree_fetches":0'), + line.replace(b'"writer_revision":1', b'"writer_revision":1,"writer_revision":1'), + line[:-2] + b"\n"] + for raw in bad: + self.write([self.payload], raw=raw) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.write([self.payload, self.payload]) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + + def test_file_and_line_caps_and_changed_record_inode_fail(self): + self.write([self.payload], raw=b"x" * (projection.FILE_BYTES + 1)) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.write([self.payload], raw=b" " * projection.RECORD_BYTES + self.line(self.payload, 1)) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.write([self.payload]) + collector = projection.ProjectionCollector(self.cache) + self.collect(collector) + replacement = self.root / "other" + replacement.write_bytes((self.root / "records.jsonl").read_bytes()) + replacement.chmod(0o600) + os.replace(replacement, self.root / "records.jsonl") + with self.assertRaises(projection.TraceRejected): + collector.snapshot() + + def test_unknown_request_bad_receipt_or_duplicate_logical_call_fail(self): + self.write([dict(self.payload, request_id="unknown:a1")]) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.write([self.payload]) + for changes in ({"retry_count": True}, {"final_attempt_id": "wrong:a1"}, + {"attempt_ids": [self.logical + ":a2"]}, {"extra": 0}): + original = self.measured["resolve_trace_receipt"] + self.measured["resolve_trace_receipt"] = dict(original, **changes) + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + self.measured["resolve_trace_receipt"] = original + collector = projection.ProjectionCollector(self.cache) + self.collect(collector) + with self.assertRaises(projection.TraceRejected): + self.collect(collector) + + @unittest.skipUnless(os.name == "posix", "actual Unix symlink and mode checks") + def test_symlink_private_modes_and_multiple_sink_directories_fail(self): + self.write([self.payload]) + self.parent.chmod(0o755) + with self.assertRaises(projection.TraceRejected): + projection.ProjectionCollector(self.cache) + self.parent.chmod(0o700) + (self.root / "records.jsonl").rename(self.root / "actual") + (self.root / "records.jsonl").symlink_to(self.root / "actual") + with self.assertRaises(projection.TraceRejected): + self.collect(projection.ProjectionCollector(self.cache)) + (self.parent / str(uuid.uuid4())).mkdir(mode=0o700) + with self.assertRaises(projection.TraceRejected): + projection.ProjectionCollector(self.cache) + + def test_explicit_dispatch_anchors_do_not_reset_on_queue_or_recovery(self): + start = datetime.now(timezone.utc) + deadline = start + timedelta(minutes=235) + anchor = projection.window_anchor(start.isoformat(), deadline.isoformat(), admission=True) + self.assertAlmostEqual(anchor - time.monotonic(), 220 * 60, delta=.1) + for first, last in ((None, None), (start.isoformat(), ""), + (start.replace(tzinfo=None).isoformat(), deadline.isoformat()), + (start.isoformat(), (deadline + timedelta(seconds=1)).isoformat())): + with self.assertRaises(projection.TraceRejected): + projection.window_anchor(first, last, admission=True) + queued = start - timedelta(minutes=16) + with self.assertRaises(projection.TraceRejected): + projection.window_anchor(queued.isoformat(), (queued + timedelta(minutes=235)).isoformat(), admission=True) + self.assertLess(projection.window_anchor(queued.isoformat(), (queued + timedelta(minutes=235)).isoformat()), anchor) + + +@unittest.skipUnless(os.name == "posix" and Path("/proc").is_dir() and shutil.which("cc"), + "actual owned Linux process and C fixture") +class OwnedProjectionDrainTests(unittest.TestCase): + def start(self, root, ignore=False): + source = root / "fixture.c" + source.write_text('''#include +#include +#include +static volatile sig_atomic_t stopped; +static void stop(int sig) { (void)sig; stopped = 1; } +int main(int argc, char **argv) { + if (argc != 3) return 2; + signal(SIGINT, ''' + ("SIG_IGN" if ignore else "stop") + '''); + FILE *stream = fopen(argv[2], "a"); if (!stream) return 3; + fputs("ready\\n", stream); fflush(stream); fsync(fileno(stream)); + while (!stopped) usleep(1000); + fputs("closed\\n", stream); fflush(stream); fsync(fileno(stream)); + fclose(stream); return 0; +} +''') + binary = root / "owned-fixture" + subprocess.run(["cc", str(source), "-o", str(binary)], check=True, timeout=10, capture_output=True) + config = root / "service.toml" + config.write_text("") + dependencies = root / "dependencies.json" + dependencies.write_text("{}") + process = budget.PinnedProcess([str(binary), "--config", str(config)], start_new_session=True, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + started = budget.process_start(process.pid) + self.addCleanup(lambda: budget.stop_group(process.pid, started, time.monotonic() + 2, process) + if process.returncode is None else None) + until = time.monotonic() + 2 + while "ready" not in config.read_text() and time.monotonic() < until: + time.sleep(.01) + self.assertIn("ready", config.read_text()) + state = {"project": "owned-test", "binary": str(binary), + "compose_sha256": projection.digest(dependencies.read_bytes())[7:], + "service": {"pid": process.pid, "pgid": process.pid, "sid": process.pid, "starttime": started}} + (root / "owned.json").write_text(json.dumps(state)) + return process, config, state + + def test_actual_owned_sigint_drain_pins_leader_until_final_group_check_and_reap(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + process, config, state = self.start(root) + self.assertIsNone(ci.owned_service_exit(process)) + self.assertIsNone(process.returncode) + ci.graceful_owned(root, state["project"], time.monotonic() + 2, process) + self.assertEqual(process.returncode, 0) + self.assertIn("closed", config.read_text()) + self.assertNotIn("service", json.loads((root / "owned.json").read_text())) + + def test_changed_ownership_refuses_sigint_and_ignored_sigint_has_no_new_deadline(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + process, config, state = self.start(root, ignore=True) + with self.assertRaises(AssertionError): + ci.graceful_owned(root, "other-project", time.monotonic() + 1, process) + self.assertIsNone(ci.owned_service_exit(process)) + started = time.monotonic() + with self.assertRaises(TimeoutError): + ci.graceful_owned(root, state["project"], started + .05, process) + self.assertLess(time.monotonic() - started, .3) + self.assertNotIn("closed", config.read_text()) + budget.stop_group(process.pid, state["service"]["starttime"], time.monotonic() + 1, process) + + +if __name__ == "__main__": + unittest.main()