diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d61fd..f12a29b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: env: TOKENIZERS_PARALLELISM: 'false' OMP_NUM_THREADS: '2' + GLOO_SOCKET_IFNAME: ${{ matrix.os == 'macos-15' && 'lo0' || 'lo' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 @@ -40,8 +41,10 @@ jobs: run: | for iteration in $(seq 1 50); do echo "CPU owner/fixture repetition $iteration/50" - uv run --no-sync python -m pytest -q tests/test_cache_retirement.py \ + uv run --no-sync python -m pytest -q tests/test_cache_retirement.py tests/test_execution_retirement.py tests/test_cache_identity.py \ tests/test_distributed_http.py::test_rank_fixture_joins_watcher_before_destroying_process_groups \ + tests/test_distributed_http.py::test_failed_runner_retention_does_not_leave_fixture_gloo_threads_alive \ + tests/test_distributed_http.py::test_distributed_http_routes_real_replica_engines[fatal-2] \ tests/test_deepseek_auxiliary_graphs.py::test_auxiliary_callbacks_bind_model_projections_and_preserve_output_dtypes \ models/cuda/tests/test_deepseek_v41_projections.py::test_prepared_vocab_head_preserves_last_row_full_logits_and_fp32_output done diff --git a/crates/orbitinfer-core/src/engine.rs b/crates/orbitinfer-core/src/engine.rs index c432542..a292d87 100644 --- a/crates/orbitinfer-core/src/engine.rs +++ b/crates/orbitinfer-core/src/engine.rs @@ -17,6 +17,7 @@ use crate::{ Phase, PlanningConfig, PlanningError, RequestId, RequestProgress, Schedulable, Scheduler, Selection, SequenceStep, TokenRange, schedule_ready_prefix, }; +use std::any::Any; use std::cmp::Reverse; use std::fmt; use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; @@ -546,6 +547,7 @@ struct Batch

{ } pub struct EngineCore

{ + backing_owner: Option>, metrics: Option, in_flight: Option>, continuation: Option>, @@ -567,13 +569,35 @@ pub struct EngineCore

{ closed: bool, } +impl

Drop for EngineCore

{ + fn drop(&mut self) { + if self.in_flight.is_some() || self.quarantined.is_some() || self.poisoned_error.is_some() { + // Drop is not a device receipt. Retain actual backing together with + // the bounded batch, logical page charges and storage borrowers. + // No cleanup is retried after an uncertain native failure. + std::mem::forget(self.backing_owner.take()); + std::mem::forget(self.in_flight.take()); + std::mem::forget(self.quarantined.take()); + std::mem::forget(self.continuation.take()); + std::mem::forget(self.kv_resources.take()); + } + } +} + impl EngineCore

{ + /// Bind the actual arena/runtime owner to the logical page allocator. + /// + /// `backing_owner` must retain every allocation and runtime resource the + /// runner can access, including allocations installed by resource binding. + /// Prepared work and uncertain failure keep this owner even after core drop. + /// Complete every accepted step and call `close` for orderly retirement. pub fn new( runner_config: RunnerConfig, max_running_requests: usize, context_length: u32, scheduler_config: SchedulerConfig, prefix_cache: bool, + backing_owner: Arc, ) -> Result { let max_running_requests = NonZeroUsize::new(max_running_requests) .ok_or_else(|| Error::invalid("max_running_requests must be positive"))?; @@ -617,6 +641,7 @@ impl EngineCore

{ let new_token_ratio_decay = (new_token_ratio - minimum_new_token_ratio) / NEW_TOKEN_RATIO_DECAY_STEPS; Ok(Self { + backing_owner: Some(backing_owner), metrics: None, in_flight: None, continuation: None, diff --git a/crates/orbitinfer-core/tests/engine.rs b/crates/orbitinfer-core/tests/engine.rs index 1298960..5f5833f 100644 --- a/crates/orbitinfer-core/tests/engine.rs +++ b/crates/orbitinfer-core/tests/engine.rs @@ -93,6 +93,7 @@ fn engine(lookahead: u32, chunk: u32) -> EngineCore<&'static str> { ..Default::default() }, false, + std::sync::Arc::new(()), ) .unwrap() } @@ -125,6 +126,7 @@ fn admission_credits_restored_physical_prefix_pages_only() { ..Default::default() }, true, + std::sync::Arc::new(()), ) .unwrap(); engine.enable_metrics().unwrap(); @@ -213,6 +215,7 @@ fn check_stopped_speculative_prefix(stop_token: bool) { 32, SchedulerConfig::default(), true, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model { @@ -531,6 +534,7 @@ fn panicking_checkpoint_access_quarantines_the_accepted_transaction() { 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap() }; @@ -584,6 +588,7 @@ fn malformed_output_and_receipt_panic_quarantine_before_any_result_is_published( 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap() }; @@ -664,6 +669,7 @@ fn a_panicking_payload_clone_restores_the_cohort_and_releases_its_cache_access() 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut handles = Vec::new(); @@ -807,6 +813,7 @@ fn decode_continuation_transfers_writer_lease_and_commits_exact_feedback_depth() 512, SchedulerConfig::default(), true, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -868,6 +875,7 @@ fn unsafe_decode_boundaries_fall_back_before_enqueuing_a_continuation() { 512, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -916,6 +924,7 @@ fn cross_page_engine(prefix_cache: bool) -> EngineCore<&'static str> { 256, SchedulerConfig::default(), prefix_cache, + std::sync::Arc::new(()), ) .unwrap() } @@ -1029,6 +1038,7 @@ fn shifted_decode_engine() -> Result, Error> { 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) } @@ -1089,6 +1099,7 @@ fn accepted_limit_defaults_to_one_and_rejects_zero_or_overflowing_write_footprin 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ); assert_eq!(result.err().unwrap().kind, ErrorKind::InvalidInput); } @@ -1114,6 +1125,7 @@ fn shifted_write_guard_survives_a_near_context_limit_without_changing_outputs() 128, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); assert!(engine.submit(input(vec![1; 126], 4)).is_err()); @@ -1210,6 +1222,7 @@ fn window_guard_bounds_automatic_reservation_and_rejects_an_oversized_write() { 256, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ); if read_window == 9 { assert_eq!(result.err().unwrap().kind, ErrorKind::InvalidInput); @@ -1253,6 +1266,7 @@ fn window_guard_bounds_automatic_reservation_and_rejects_an_oversized_write() { 256, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .is_ok() ); @@ -1299,6 +1313,7 @@ fn hybrid_reservation_engine(prefix_cache: bool) -> EngineCore<&'static str> { ..Default::default() }, prefix_cache, + std::sync::Arc::new(()), ) .unwrap() } @@ -1468,6 +1483,7 @@ fn hybrid_reservation_supports_distinct_private_state_checkpoint_cadences() { 2048, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -1612,6 +1628,7 @@ fn rounded_reservation_tail_cannot_exceed_a_windows_declared_write_guard() { 256, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -1654,6 +1671,7 @@ fn bounded_reservation_engine(capacity: u32, context: u32) -> EngineCore<&'stati ..Default::default() }, false, + std::sync::Arc::new(()), ) .unwrap() } @@ -1849,6 +1867,7 @@ fn stop_overlap_engine() -> EngineCore<&'static str> { 512, SchedulerConfig::default(), true, + std::sync::Arc::new(()), ) .unwrap() } @@ -1988,6 +2007,7 @@ fn retraction_counts_compute_replay_without_duplicating_logical_tokens() { ..Default::default() }, false, + std::sync::Arc::new(()), ) .unwrap(); engine.enable_metrics().unwrap(); @@ -2131,6 +2151,7 @@ fn failure_after_next_enqueue_keeps_transferred_lease_quarantined() { 512, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -2173,6 +2194,7 @@ fn waiting_prefix_owners_are_retracted_before_running_requests_under_pressure() ..Default::default() }, true, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -2245,6 +2267,7 @@ fn failed_prefill_growth_releases_unadmitted_pages_before_retrying_running_work( schedule_conservativeness: 0.0, }, true, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -2304,6 +2327,7 @@ fn a_displaced_rejected_request_does_not_block_a_higher_priority_request() { 3, SchedulerConfig::default(), false, + std::sync::Arc::new(()), ) .unwrap(); let mut model = Model::default(); @@ -2328,3 +2352,58 @@ fn a_displaced_rejected_request_does_not_block_a_higher_priority_request() { assert_eq!(rejected.take_result().unwrap(), [4]); assert_eq!(engine.cache_free_parent_count(), Some(3)); } + +#[test] +fn final_core_drop_retains_actual_backing_until_execution_retirement() { + use std::sync::Arc; + for outcome in ["pending", "failed", "success", "rejected"] { + let backing = Arc::new(vec![73_u8; 64]); + let weak = Arc::downgrade(&backing); + let mut engine = EngineCore::new( + RunnerConfig { + cache_layout: Some( + KvLayout::new( + 4, + vec![KvGroupLayout::new(2, 1, KvRetention::FullContext).unwrap()], + ) + .unwrap(), + ), + ..Default::default() + }, + 1, + 32, + SchedulerConfig::default(), + false, + backing, + ) + .unwrap(); + let mut model = Model::default(); + let request = engine.submit(input(vec![1], 1)).unwrap(); + engine.prepare_step(&mut model, None).unwrap().unwrap(); + if outcome != "pending" { + let rows = owned_rows(vec![vec![2]]); + let result = match outcome { + "failed" => Err(Error::execution("device receipt failed")), + "rejected" => Err(Error::rejected("proved no access")), + _ => Ok(rows.view()), + }; + let completed = engine.complete_step_with(&mut model, result, Ok); + assert_eq!(completed.is_ok(), outcome == "success"); + } + let uncertain = matches!(outcome, "pending" | "failed"); + if uncertain { + assert!(!request.done()); + assert_eq!(engine.cache_occupied_block_count(), Some(1)); + assert!(engine.close(&mut model).is_err()); + assert!(model.released.is_empty()); + } else { + engine.close(&mut model).unwrap(); + assert_eq!(engine.cache_occupied_block_count(), Some(0)); + } + drop(engine); + assert_eq!(weak.upgrade().is_some(), uncertain); + if uncertain { + assert_eq!(weak.upgrade().unwrap().as_slice(), &[73; 64]); + } + } +} diff --git a/crates/orbitinfer-core/tests/preparation.rs b/crates/orbitinfer-core/tests/preparation.rs index 93e619f..185690b 100644 --- a/crates/orbitinfer-core/tests/preparation.rs +++ b/crates/orbitinfer-core/tests/preparation.rs @@ -102,6 +102,7 @@ fn engine(prefix: bool) -> EngineCore { ..Default::default() }, prefix, + std::sync::Arc::new(()), ) .unwrap() } diff --git a/crates/orbitinfer-core/tests/prompt_scores.rs b/crates/orbitinfer-core/tests/prompt_scores.rs index 4c30672..924242f 100644 --- a/crates/orbitinfer-core/tests/prompt_scores.rs +++ b/crates/orbitinfer-core/tests/prompt_scores.rs @@ -18,6 +18,7 @@ fn engine(chunk: u32) -> EngineCore<()> { ..Default::default() }, false, + std::sync::Arc::new(()), ) .unwrap() } diff --git a/crates/orbitinfer-py/src/lib.rs b/crates/orbitinfer-py/src/lib.rs index 1bb4439..16a2910 100644 --- a/crates/orbitinfer-py/src/lib.rs +++ b/crates/orbitinfer-py/src/lib.rs @@ -1261,6 +1261,7 @@ struct Engine { overlap_enqueues: u64, overlap_continuations: u64, runner_metrics: Option>, + prefix_storage: Option>, } struct PendingExecution { @@ -1289,6 +1290,7 @@ impl Engine { Ok(feedback) if feedback.is_none() => None, Ok(feedback) => Some(feedback), Err(error) => { + self.retain_failure_frames(py, &error); self.core.poison_message(format!( "accepted completion feedback failed: {error}" )); @@ -1318,6 +1320,7 @@ impl Engine { let result = match enqueue.bind(py).call((&next_plan,), Some(&kwargs)) { Ok(result) => result.unbind(), Err(error) => { + self.retain_failure_frames(py, &error); self.core .poison_message(format!("overlapped enqueue failed: {error}")); return Err(error); @@ -1326,6 +1329,7 @@ impl Engine { let asynchronous = match runner_result_is_async(result.bind(py)) { Ok(asynchronous) => asynchronous, Err(error) => { + self.retain_failure_frames(py, &error); self.failed_executions.push(result); self.core.poison_message(format!( "accepted completion inspection failed: {error}" @@ -1499,6 +1503,14 @@ impl Engine { }) } + fn retain_failure_frames(&mut self, py: Python<'_>, error: &PyErr) { + if let Some(traceback) = error.traceback(py) { + // Failed frames may own temporary staging/output buffers. Keep the + // existing chain, not future caller frames appended to this error. + self.failed_executions.push(traceback.into_any().unbind()); + } + } + fn fail_output(&mut self, py: Python<'_>, error: PyErr, accepted: bool) -> PyErr { let forward = self.runner_forward.clone_ref(py); let release = self @@ -1507,6 +1519,9 @@ impl Engine { .map(|value| value.clone_ref(py)); let mut runner = PythonRunner::new(py, &forward, release.as_ref()); let accepted = accepted || runner_error_is_post_accept(py, &error); + if accepted { + self.retain_failure_frames(py, &error); + } let failure = runner.failed(error, accepted); let result = self.core.complete_step_with(&mut runner, Err(failure), Ok); result.err(); @@ -1534,7 +1549,7 @@ impl Engine { #[pymethods] impl Engine { #[new] - #[pyo3(signature = (runner: "Runner", *, max_running_requests = 32, context_length = 32768, max_top_logprobs = DEFAULT_MAX_TOP_LOGPROBS, scheduler_config = None, prefix_cache = false, prefix_cache_storage: "CacheStorage | None" = None, prefix_cache_max_inflight = None, prefix_cache_storage_policy: r#"Literal["write_through", "write_through_selective"]"# = "write_through_selective", enable_overlap = true, enable_metrics = true))] + #[pyo3(signature = (runner: "Runner", *, max_running_requests = 32, context_length = 32768, max_top_logprobs = DEFAULT_MAX_TOP_LOGPROBS, scheduler_config = None, prefix_cache = false, prefix_cache_storage: "CacheStorage | None" = None, prefix_cache_max_inflight = None, prefix_cache_identity: "CacheIdentity | None" = None, prefix_cache_max_bytes = None, prefix_cache_storage_policy: r#"Literal["write_through", "write_through_selective"]"# = "write_through_selective", enable_overlap = true, enable_metrics = true))] #[allow( clippy::too_many_arguments, reason = "Python constructor exposes keyword configuration" @@ -1549,6 +1564,8 @@ impl Engine { prefix_cache: bool, prefix_cache_storage: Option>, prefix_cache_max_inflight: Option, + prefix_cache_identity: Option>, + prefix_cache_max_bytes: Option, prefix_cache_storage_policy: &str, enable_overlap: bool, enable_metrics: bool, @@ -1593,6 +1610,7 @@ impl Engine { schedule_conservativeness: scheduler.schedule_conservativeness, }, prefix_cache, + Arc::new(runner.clone_ref(py)), ) .map_err(core_error)?; core.set_max_top_logprobs(max_top_logprobs); @@ -1610,11 +1628,19 @@ impl Engine { } else { None }; + let mut prefix_storage = None; if let Some(provider) = prefix_cache_storage { let adapter = PyModule::import(py, "orbitinfer._cache_storage")? .getattr("_PrefixStorage")? - .call1((provider, runner, &config.cache_layout))? + .call1(( + provider, + runner, + &config.cache_layout, + prefix_cache_identity, + prefix_cache_max_bytes, + ))? .unbind(); + prefix_storage = Some(adapter.clone_ref(py)); core.set_prefix_storage( Box::new(cache_storage::PythonPrefixStorage(adapter)), prefix_cache_max_inflight, @@ -1636,6 +1662,7 @@ impl Engine { overlap_enqueues: 0, overlap_continuations: 0, runner_metrics, + prefix_storage, }) } #[pyo3(signature = ( @@ -1980,6 +2007,15 @@ impl Engine { self.core.cache_occupied_block_count() } + #[getter] + /// Borrowed-byte charges are returned only after a successful storage receipt. + fn prefix_storage_stats(&self, py: Python<'_>) -> PyResult>> { + self.prefix_storage + .as_ref() + .map(|storage| storage.bind(py).call_method0("stats").map(Bound::unbind)) + .transpose() + } + /// `(entries, hits, misses, evictions)` when prefix caching is enabled. #[getter] fn prefix_cache_stats(&self) -> Option<(usize, u64, u64, u64)> { @@ -2006,19 +2042,22 @@ impl Engine { impl Drop for Engine { fn drop(&mut self) { - if self.core.closed() || self.core.poisoned() { - return; - } - - match Python::try_attach(|py| self.close(py)) { - Some(Ok(())) => {} - Some(Err(error)) => { - self.core - .poison_message(format!("Engine destructor cleanup failed: {error}")); + if !self.core.closed() && !self.core.poisoned() { + match Python::try_attach(|py| self.close(py)) { + Some(Ok(())) => {} + Some(Err(error)) => self + .core + .poison_message(format!("Engine destructor cleanup failed: {error}")), + None => self.core.poison_message( + "Engine destructor could not attach to Python; resources remain quarantined", + ), } - None => self.core.poison_message( - "Engine destructor could not attach to Python; resources remain quarantined", - ), + } + if self.core.poisoned() { + // The core retains the runner's backing and page charges. These + // completion objects can own additional device/host work buffers. + std::mem::forget(self.pending.take()); + std::mem::forget(std::mem::take(&mut self.failed_executions)); } } } diff --git a/docs/architecture.md b/docs/architecture.md index ca040c0..a0b5641 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,17 +12,17 @@ alternative providers. These integrations are not implemented yet. The [hierarchical cache proposal](external-kv-design.md) extends the existing storage owners without adding a second scheduler or allocator. -Cancellation and logical completion do not prove physical retirement. Before -enabling an installed profile, prove that actual arena backing, Python owners -and registrations survive failed device completion and final-reference drop. -The imported quarantine controls alone do not establish that proof. +Cancellation and logical completion do not prove physical retirement. CPU controls retain actual arena backing, Python owners and temporary +completion buffers across uncertain execution and final-reference drop. Installed +GPU/model profiles still need device-fault and drain validation. External storage now reports logical outcome and borrowed-buffer retirement separately through `CacheTransfer`. The native storage job stays charged until the binding observes a successful retirement receipt. Failed receipts retain exact exported views and fence that storage binding, including after final -Engine drop. This proves the Python storage boundary on CPU; it does not close -the remaining native-runner or installed GPU lifetime audit. +Engine drop. Identity includes explicit weights, semantics, dtype, format and shard authority +plus the ordered cache geometry. Borrowed bytes remain charged until retirement. +These controls prove the Rust/Python ownership boundary on CPU, not GPU execution. ## Runtime owners @@ -95,9 +95,15 @@ public API that commits arbitrary page IDs. Rust runners implement `engine::Runner

` and receive an owned `ExecutionPlan

`. The native [conformance tests](../crates/orbitinfer-core/tests/engine.rs) demonstrate this API without a Python interpreter. Native applications bind their arena to the core's -cache pool and use the same logical runner throughout its lifetime. They call -`close(&mut runner)` for orderly shutdown; the core does not own the device -runtime or call back into a runner when it is dropped. +cache pool and use the same logical runner throughout its lifetime. Pass an `Arc` owning the actual arena and runtime to `EngineCore::new`; +the owner must include allocations added by resource binding and transient work. +Call `close(&mut runner)` for orderly shutdown. Core drop cannot call the runner: a +prepared or quarantined transaction retains its backing owner, page charges and +accepted payloads. The Python binding supplies the runner itself and additionally +retains uncertain completion objects and existing failure tracebacks (which may +own temporary staging). The poisoned engine admits no further execution. This +intentional quarantine is bounded by the engine's admitted work; it is not a +recovery mechanism or permission to retry state mutation. Multiple requests within a batch execute together. With device overlap, an initial `step()` can return no events while its batch runs; subsequent steps diff --git a/docs/cache-storage.md b/docs/cache-storage.md index b978b37..405df7e 100644 --- a/docs/cache-storage.md +++ b/docs/cache-storage.md @@ -7,8 +7,8 @@ a storage provider keeps immutable checkpoints outside the custom providers can implement other storage tiers through the same interface. The [hierarchical external KV proposal](external-kv-design.md) specifies identity, -byte budgets and the planned OrbitKV/Mooncake Store/LMCache adapters. Independent -operation and retirement receipts are implemented below. The remaining proposal +byte budgets and the planned OrbitKV/Mooncake Store/LMCache adapters. Identity, +budgets and independent operation/retirement receipts are implemented below. The remaining proposal is not an integration claim; none of those three adapters is available yet. ## Using the DRAM cache @@ -17,7 +17,7 @@ Construct a provider and pass it to an Engine with prefix caching enabled. Given runner and prompt token IDs: ```python -from orbitinfer import Engine, MemoryCacheStorage, SamplingParams +from orbitinfer import CacheIdentity, Engine, MemoryCacheStorage, SamplingParams storage = MemoryCacheStorage(max_bytes=8 * 1024**3) # 8 GiB try: @@ -25,6 +25,12 @@ try: runner, prefix_cache=True, prefix_cache_storage=storage, + prefix_cache_identity=CacheIdentity( + model="model-id", weights="immutable-weight-revision", + format="runner-cache-v1", dtype="bfloat16", shard="tp=0/1,pp=0/1", + semantics="attention-rope-adapter-fingerprint", + ), + prefix_cache_max_bytes=256 * 1024**2, # Outstanding borrowed views: 256 MiB ) as engine: request = engine.submit( token_ids, @@ -41,6 +47,9 @@ finally: pinned host memory when CUDA is available. Its LRU budget charges object keys, region names and payload bytes; general Python and allocator overhead is outside that budget. One background worker owns a private CUDA copy stream per device. +Its queue defaults to 64 transfers (`max_inflight`), with borrowed bytes limited +to `max_bytes * max_inflight` (`max_pending_bytes` can set a smaller limit). +Rejection returns a false outcome with a successful no-access receipt. It queues pending reads before fencing the stream and completing their futures. Writes retire earlier reads before evicting or reusing their host memory. @@ -78,8 +87,9 @@ same top-level field. Reuse requires the same salt and matching text or media identity, including when restoring checkpoints from a shared storage provider. The salt changes cache keys, not prompt tokens or generated output. -Omitting `cache_salt` or passing `None` (`null` over HTTP) preserves the existing -unsalted namespace and storage keys. An empty string is a separate namespace. +Omitting `cache_salt` or passing `None` (`null` over HTTP) selects the unsalted +namespace. All external keys now include the representation identity; old +unbound keys intentionally miss. An empty string is a separate namespace. Other values must be strings. Salts select cache namespaces; they do not provide authentication or access control. @@ -211,3 +221,25 @@ intersecting page rather than only individual live rows. The hook must accept empty groups with `prefix_length=0` and return no blocks. The Engine uses this capability check during construction. Missing or unsupported exporters raise `NotImplementedError` before resources are bound. + +## Representation and transfer bounds + +External storage requires `prefix_cache_identity=CacheIdentity(...)` and a positive +`prefix_cache_max_bytes`. The caller supplies authoritative model/weight, format, +stored dtype, semantic fingerprint and logical shard identity. Do not use mutable +weight aliases or physical GPU numbers. Identical replicas may share identity; +different tensor shards must not. The binding hashes this immutable value with +ordered cache group geometry, replay suffix and decode write span, then the +length-delimited request namespace and token chain. Arena capacity is excluded. +A mismatched identity misses before any destination copy; no implicit conversion +or resharding is performed. + +All exports are validated before provider submission: contiguous byte views, +bounded names and region counts, total bytes and non-overlapping load destinations. +The limit is 65,536 keys/blocks/regions per operation and 1,024 UTF-8 bytes per name. +`prefix_cache_max_bytes` charges every exported view, including aliased read-only +sources, until **both** outcome and a successful retirement receipt are observed. +An oversized or over-budget operation posts nothing and falls back to prefill. +A failed receipt keeps its charge and fences the binding. Inspect +`engine.prefix_storage_stats` for current, peak and quarantined bytes, rejection +count and last preflight reason. Provider-resident DRAM has its separate budget. diff --git a/docs/completion-plan.md b/docs/completion-plan.md index d455eb9..df2750c 100644 --- a/docs/completion-plan.md +++ b/docs/completion-plan.md @@ -21,10 +21,11 @@ paths inherited by subprocess controls. Current Clippy's nonzero-divisor checks are addressed without changing scheduling arithmetic. Keep this CI baseline passing before adding providers. -P0 remains open: the Python storage path now retains exact exported views on -failed retirement and final-reference drop, with forced-interleaving controls. -Next audit native/device execution backing and enforce representation identity -and borrowed-byte bounds. P3 begins with the CPU provider contract and OrbitKV +P0 is implemented at the Rust/Python owner boundary: final drops retain actual +backing, uncertain completions and page charges; storage keys bind representation +identity and borrowed bytes are bounded until retirement. CPU controls include +synchronous failure, asynchronous inspection/harvest failure and a successful +release control. Installed CUDA execution remains unqualified. P3 begins with the CPU provider contract and OrbitKV DRAM integration; SSD and peer profiles follow separately. GPU integration still needs installed validation. The first deployment target remains DeepSeek on SM100. Other imported runners are available for separate validation. @@ -32,7 +33,7 @@ SM100. Other imported runners are available for separate validation. | Phase | Scope | State | | --- | --- | --- | | Foundation | Complete runtime, all model packages, tooling, tests and provenance | Source imported; validation in source-import.md; clean CPU gate repairs included | -| P0 | Final-reference/failed-completion lifetime audit, existing storage integration, real backing, identity and byte bounds | Partial: explicit Python storage receipts and backing-retention controls; native/device audit, identity and byte bounds remain open | +| P0 | Final-reference/failed-completion lifetime audit, existing storage integration, real backing, identity and byte bounds | Implemented at source level; real backing, identity, byte and metadata bounds verified on CPU; installed CUDA gate unrun | | P1 | Full-checkpoint offline DeepSeek numerical comparison on SM100 | Unrun; hardware required | | P2 | CUDA graphs, overlap, persistent-state faults and cancellation | Source imported; installed profile unqualified | | P3 | OrbitKV DRAM, then SSD/peer through OrbiTE; separate Mooncake Store and LMCache adapters | Design proposed; not implemented; sequence in external-kv-design.md | @@ -58,3 +59,9 @@ and actual accepted tokens; forced draft acceptance is not throughput evidence. The imported deployment scripts and skills are tools, not authorization to run cloud jobs, download complete checkpoints or publish packages. OrbitKV and OrbiTE integrations remain separate work. + +### External-cache installed gate + +`uv run --no-sync python tests/hardware/cache_storage_cuda.py --device cuda:0` +checks GPU views, guards, prefix restore and final-owner quarantine on SM100. +Written but not run. It does not qualify full-model execution or OrbitKV. diff --git a/docs/external-kv-design.md b/docs/external-kv-design.md index b092f4a..55a127b 100644 --- a/docs/external-kv-design.md +++ b/docs/external-kv-design.md @@ -2,7 +2,7 @@ Design and implementation sequence. The current interface and independent retirement receipts are described in [prefix cache storage](cache-storage.md). -Identity binding, byte budgets and the provider adapters remain proposed work. +Identity binding and borrowed-byte budgets are implemented; provider adapters remain the next step. No OrbitKV, Mooncake Store or LMCache adapter is implemented yet. ## Scope @@ -62,8 +62,8 @@ not a cache provider or a global directory. | Existing owner | Keep | Work required before integration | | --- | --- | --- | | `EngineCore` and `PagedKvResources` | Prefix matching, page allocation, storage jobs and commit | Prove final-reference retention of actual backing; add byte accounting for external operations | -| `PrefixStorage` / `PrefixStorageOperation` | Native lookup/transfer/poll boundary; Python outcome/retirement receipts | Complete the remaining native-runner backing audit without a second progress loop | -| `_PrefixStorage` / `CacheStorage` | `lookup`, `load`, `store`; immutable objects | Bind representation identity and validate provider retirement semantics | +| `PrefixStorage` / `PrefixStorageOperation` | Native lookup/transfer/poll boundary; Python outcome/retirement receipts | Core retains an explicit backing owner; Python also retains uncertain completions and temporary failure frames | +| `_PrefixStorage` / `CacheStorage` | `lookup`, `load`, `store`; immutable objects | Explicit CacheIdentity plus cache geometry; bound borrowed bytes and metadata until retirement | | `cache_storage_buffers` | Runner's named region views, no export-time copy | Retain arena and registration owners, generation and producer readiness explicitly | | `MemoryCacheStorage` | Bounded DRAM reference provider and existing controls | Audit final-owner and failed-fence paths before declaring the new contract complete | @@ -142,7 +142,7 @@ retirement futures. A logical exception cannot release buffers. The binding returns a terminal native transfer result only after a successful retirement receipt; failed retirement is sticky, retains the actual exported views and fences new storage work. This is tested through the real Engine, including final -reference drop. The wider native-runner and GPU execution audit remains in P0. +reference drop. Native final-reference controls retain the arena owner and charges; installed GPU faults remain unqualified. For load, acquire private destination write leases before accepting physical work, and order copies after source readiness. Make the restored pages visible @@ -266,7 +266,7 @@ must force interleavings and inspect backing and budget receipts. 1. **P0: ownership and identity.** Audit the current Rust/Python path, retain actual arena backing across final drops, and add identity and byte bounds. The Python storage receipt/view-retention controls are implemented. Native - execution backing, identity binding and byte accounting remain open. + execution backing, identity binding and byte accounting now have CPU controls. Installed device validation is still required. 2. **P3a: OrbitKV local DRAM.** Add the thin adapter and a CPU contract test. Write the SM100 runner export/producer/readiness gate; run it only on an explicitly authorized installed host. Do not modify OrbitKV from this work. diff --git a/docs/kernel-boundary.md b/docs/kernel-boundary.md index 8036847..e2bbb2c 100644 --- a/docs/kernel-boundary.md +++ b/docs/kernel-boundary.md @@ -129,3 +129,15 @@ failure quarantines resources even if CUDA synchronization succeeds: an in-place recurrent-state or sliding-window update cannot be undone by waiting for the device. Graph host-input validation remains retryable because it occurs before device copies or model execution. + +## Final owner drop + +Native applications pass an `Arc` with the arena/runtime owner to +`EngineCore::new`. It must include every allocation the runner can access, including +transient work. `Error::rejected` is valid only when no persistent state changed +and no outstanding access remains. An uncertain failure poisons execution and +retains the owner and logical charges after core drop. Python retains the bound +runner, completion objects and temporary failure frames. This is containment, +not retirement: call `close` after successfully harvesting every accepted step +for ordinary release. An installed device profile must verify these receipts on +its runtime; CPU controls do not prove GPU quiescence. diff --git a/docs/releases.md b/docs/releases.md index 5d9e4de..a82b02d 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -2,6 +2,16 @@ ## Unreleased +- External storage now requires `CacheIdentity` and `prefix_cache_max_bytes` at + Engine construction. Keys include weights, format, dtype, semantics, shard and + cache geometry; older keys intentionally miss. Invalid/overlapping exports and + byte-limit rejection never call the provider. `prefix_storage_stats` reports + borrowed and quarantined bytes. MemoryCacheStorage also bounds queued work. +- Native `EngineCore::new` now requires an `Arc` owning actual execution backing. + Prepared or failed execution retains this owner on final drop. Python supplies + its runner and retains uncertain completion/temporary owners. Successful close + still releases them; no uncertain execution is replayed. + - External `CacheStorage.load` and `store` now return `CacheTransfer`, with separate `outcome` and `retirement` futures, instead of a bool/None future. Update providers to publish a successful retirement receipt only after all diff --git a/orbitinfer/__init__.py b/orbitinfer/__init__.py index db906c3..97d7a3f 100644 --- a/orbitinfer/__init__.py +++ b/orbitinfer/__init__.py @@ -33,6 +33,7 @@ __version__, ) from orbitinfer.cache_storage import ( + CacheIdentity, CacheRetirementError, CacheStorage, CacheTransfer, @@ -48,6 +49,7 @@ __all__ = [ "CacheExporter", + "CacheIdentity", "CacheStorage", "CacheTransfer", "CacheRetirementError", diff --git a/orbitinfer/_cache_storage.py b/orbitinfer/_cache_storage.py index ec5d6ea..ee59ac4 100644 --- a/orbitinfer/_cache_storage.py +++ b/orbitinfer/_cache_storage.py @@ -2,13 +2,19 @@ """Engine-owned prefix matching over asynchronous opaque byte storage.""" import hashlib +import json import math +from collections.abc import Mapping from concurrent.futures import Future +from dataclasses import asdict import numpy as np from orbitinfer._native import CacheGroupFamily -from orbitinfer.cache_storage import CacheTransfer +from orbitinfer.cache_storage import CacheIdentity, CacheTransfer + +_MAX_REGIONS = 65536 +_MAX_REGION_NAME = 1024 class _Operation: @@ -29,12 +35,14 @@ def poll(self, wait=False): class _TransferOperation: """Retain actual exported views until the provider proves physical retirement.""" - def __init__(self, owner, buffers): - self.owner, self.buffers = owner, buffers + def __init__(self, owner, buffers, charge): + self.owner, self.buffers, self.charge = owner, buffers, charge self.transfer = None self.unquiesced = False def quarantine(self): + if not self.unquiesced: + self.owner.quarantined_bytes += self.charge self.unquiesced = True self.owner.halted = True @@ -55,6 +63,8 @@ def poll(self, wait=False): outcome = self.transfer.outcome if not wait and not outcome.done(): return None + self.owner.borrowed_bytes -= self.charge + self.charge = 0 # Repeated polls never refund the same owner twice. try: # Inspect failure without re-raising a stored exception: its traceback # would otherwise retain this operation and its retired buffer views. @@ -67,7 +77,7 @@ def poll(self, wait=False): class _PrefixStorage: - def __init__(self, store, runner, layout): + def __init__(self, store, runner, layout, identity, max_bytes): self.runner = runner buffers = getattr(runner, "cache_storage_buffers", None) if not callable(buffers): @@ -78,7 +88,40 @@ def __init__(self, store, runner, layout): self.halted = False if layout is None: raise ValueError("prefix storage requires a managed cache layout") + if not isinstance(identity, CacheIdentity): + raise ValueError("prefix_cache_identity must be an explicit CacheIdentity") + if type(max_bytes) is not int or not 0 < max_bytes <= (1 << 63) - 1: + raise ValueError( + "prefix_cache_max_bytes must be a positive signed 64-bit byte limit" + ) + self.max_bytes = max_bytes + self.borrowed_bytes = self.peak_bytes = self.quarantined_bytes = ( + self.rejected + ) = 0 + self.last_rejection = "" self.groups = layout.groups + geometry = { + "groups": [ + [ + g.block_tokens, + g.blocks_per_slab, + int(g.family), + int(g.retention), + g.window_tokens, + g.read_window_tokens, + ] + for g in self.groups + ], + "replay_suffix": layout.replay_suffix_tokens, + "decode_extra": layout.decode_write_extra_tokens, + } + self.identity = hashlib.sha256( + json.dumps( + ["orbitinfer-cache-1", asdict(identity), geometry], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).digest() self.alignment = math.lcm(*(g.block_tokens for g in self.groups)) self.widths = tuple(self.alignment // g.block_tokens for g in self.groups) self.history = { @@ -91,20 +134,44 @@ def __init__(self, store, runner, layout): def _keys(self, key): namespace, tokens = key encoded, width = np.asarray(tokens, dtype=" _MAX_REGIONS: + raise ValueError("cache key count exceeds metadata budget") + digest, keys = hashlib.sha256(self.identity), [] + namespace = np.asarray(namespace or (), dtype=" self.max_bytes - self.borrowed_bytes: + return self._reject("borrowed cache byte budget exhausted") + self.borrowed_bytes += charge + self.peak_bytes = max(self.peak_bytes, self.borrowed_bytes) + operation = _TransferOperation(self, buffers, charge) try: transfer = (self.store.load if load else self.store.store)( tuple(keys), buffers @@ -166,3 +212,61 @@ def transfer(self, key, groups, load): # exact views even if it did not return an operation handle. operation.quarantine() return operation + + def _export(self, key, groups, load): + import torch + + _, tokens = key + bases = self._keys(key) + if not bases or len(groups) != len(self.groups): + raise ValueError( + "cache transfer requires an aligned checkpoint and every group" + ) + if sum(len(rows) for _, rows in groups) > _MAX_REGIONS: + raise ValueError("cache block count exceeds metadata budget") + regions = iter(self.buffers(groups, prefix_length=len(tokens))) + chunks, tail, intervals, charge, count = [{} for _ in bases], {}, {}, 0, 0 + for i, (first, rows) in enumerate(groups): + for j in range(len(rows)): + target = chunks[j // self.widths[i]] if i in self.history else tail + exported = next(regions) + if not isinstance(exported, Mapping): + raise TypeError("cache block must be a mapping of named regions") + for name, region in exported.items(): + count += 1 + if count > _MAX_REGIONS: + raise ValueError("cache region count exceeds metadata budget") + if ( + not isinstance(name, str) + or not 0 < len(name.encode()) <= _MAX_REGION_NAME + ): + raise ValueError( + "cache region name must be 1..1024 UTF-8 bytes" + ) + if not isinstance(region, torch.Tensor): + raise TypeError("cache regions must be tensors") + if region.dtype != torch.uint8 or not region.is_contiguous(): + raise ValueError("cache regions must be contiguous uint8 views") + # Charge each borrowed view, even if read-only sources alias. + charge += region.nbytes + if charge > self.max_bytes: + raise ValueError("cache transfer exceeds borrowed byte budget") + if load and region.nbytes: + intervals.setdefault(region.device, []).append( + (region.data_ptr(), region.data_ptr() + region.nbytes) + ) + target[f"{i}/{first + j}/{name}"] = region + if next(regions, None) is not None: + raise ValueError("cache export returned too many blocks") + for spans in intervals.values(): + spans.sort() + if any( + end > next_start for (_, end), (next_start, _) in zip(spans, spans[1:]) + ): + raise ValueError("cache load destinations overlap") + keys, buffers = ( + ([key + b"h" for key in bases], chunks) if self.history else ([], []) + ) + keys.append(bases[-1] + b"t") + buffers.append(tail) # Empty tail still authenticates the checkpoint. + return tuple(keys), tuple(buffers), charge diff --git a/orbitinfer/_native.pyi b/orbitinfer/_native.pyi index 6f59bc8..92673b3 100644 --- a/orbitinfer/_native.pyi +++ b/orbitinfer/_native.pyi @@ -1,7 +1,7 @@ # Adapted for OrbitInfer; see NOTICE and docs/source-import.md. from collections.abc import Sequence from typing import Any, Final, final -from orbitinfer.cache_storage import CacheStorage +from orbitinfer.cache_storage import CacheIdentity, CacheStorage from typing import Literal from orbitinfer.recipe import Recipe from orbitinfer.recipe import ContentBlock @@ -507,6 +507,8 @@ class Engine: prefix_cache: bool = False, prefix_cache_storage: "CacheStorage | None" = None, prefix_cache_max_inflight: int | None = None, + prefix_cache_identity: CacheIdentity | None = None, + prefix_cache_max_bytes: int | None = None, prefix_cache_storage_policy: 'Literal["write_through", "write_through_selective"]' = "write_through_selective", enable_overlap: bool = True, enable_metrics: bool = True, @@ -584,6 +586,13 @@ class Engine: @property def poisoned(self, /) -> bool: ... @property + def prefix_storage_stats(self, /) -> dict[str, int | bool | str] | None: + """Borrowed bytes, peak, quarantined bytes and preflight rejections. + + Identity and a positive byte limit are required when binding external + storage. Failed retirement retains its charge and fences this binding. + """ + @property def prefix_cache_stats(self, /) -> tuple[int, int, int, int] | None: """ `(entries, hits, misses, evictions)` when prefix caching is enabled. diff --git a/orbitinfer/cache_storage.py b/orbitinfer/cache_storage.py index bfd45ec..c49c39d 100644 --- a/orbitinfer/cache_storage.py +++ b/orbitinfer/cache_storage.py @@ -5,7 +5,7 @@ from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import dataclass +from dataclasses import dataclass, fields from threading import Lock from time import monotonic from typing import TYPE_CHECKING, Protocol @@ -16,6 +16,34 @@ Regions = Mapping[str, "Tensor"] +@dataclass(frozen=True, slots=True) +class CacheIdentity: + """Authority for reusing external KV bytes; every field is required. + + Use immutable weight and format revisions, the stored dtype, and logical + shard coordinates (TP/PP rank and world sizes). `semantics` identifies + attention/position/adapter settings that change KV values. Each UTF-8 field + is 1..1024 bytes. Arena capacity and physical device numbers are not identity. + The Engine additionally hashes its ordered cache geometry. Incorrect caller + identity cannot be repaired by inspecting opaque bytes. + """ + + model: str + weights: str + format: str + dtype: str + shard: str + semantics: str + + def __post_init__(self): + for field in fields(self): + value = getattr(self, field.name) + if not isinstance(value, str) or not 1 <= len(value.encode()) <= 1024: + raise ValueError( + f"cache identity {field.name} must be 1..1024 UTF-8 bytes" + ) + + class CacheRetirementError(RuntimeError): """Borrowed memory may still be accessed; retain owners and stop admission.""" @@ -90,11 +118,19 @@ def store( class MemoryCacheStorage: - def __init__(self, max_bytes): + def __init__(self, max_bytes, *, max_inflight=64, max_pending_bytes=None): import torch if max_bytes < 0: raise ValueError("max_bytes must be nonnegative") + if type(max_inflight) is not int or max_inflight <= 0: + raise ValueError("max_inflight must be positive") + if max_pending_bytes is None: + max_pending_bytes = max(1, max_bytes) * max_inflight + if type(max_pending_bytes) is not int or max_pending_bytes < 0: + raise ValueError("max_pending_bytes must be nonnegative") + self.max_inflight, self.max_pending_bytes = max_inflight, max_pending_bytes + self.pending_count = self.pending_bytes = 0 self.max_bytes, self.nbytes = max_bytes, 0 self.objects, self.executor = OrderedDict(), ThreadPoolExecutor(1) self.pool = torch.empty( @@ -290,9 +326,37 @@ def _submit(self, action, buffers, finish=lambda value: value, *, defer=False): if self.closed: raise RuntimeError("cache storage is closed") - devices = {r.device for b in buffers for r in b.values() if r.is_cuda} - ready = {d: torch.cuda.current_stream(d).record_event() for d in devices} + charge = sum(r.nbytes for b in buffers for r in b.values()) transfer = CacheTransfer(Future(), Future()) + with self._lock: + if self.closed: + raise RuntimeError("cache storage is closed") + if ( + self.halted + or self.pending_count >= self.max_inflight + or charge > self.max_pending_bytes - self.pending_bytes + ): + transfer.outcome.set_result(False) + transfer.retirement.set_result(None) # Nothing was enqueued. + return transfer + self.pending_count += 1 + self.pending_bytes += charge + + def release(receipt): + if receipt.cancelled() or receipt.exception() is not None: + return # Failed fences keep both queued count and bytes charged. + with self._lock: + self.pending_count -= 1 + self.pending_bytes -= charge + + transfer.retirement.add_done_callback(release) + try: + devices = {r.device for b in buffers for r in b.values() if r.is_cuda} + ready = {d: torch.cuda.current_stream(d).record_event() for d in devices} + except BaseException as error: + transfer.outcome.set_exception(error) + transfer.retirement.set_result(None) # No copy or stream wait was issued. + return transfer # Consumers may cancel a queued outcome, but may not cancel the owner's # retirement accounting. Only run()/the device fence produces its receipt. transfer.retirement.set_running_or_notify_cancel() @@ -326,10 +390,14 @@ def run(): self._retire_queued = True self.executor.submit(self._retire_task) - with self._lock: - if self.closed: - raise RuntimeError("cache storage is closed") - self.executor.submit(run) + try: + with self._lock: + if self.closed: + raise RuntimeError("cache storage is closed") + self.executor.submit(run) + except BaseException as error: + transfer.outcome.set_exception(error) + transfer.retirement.set_result(None) # The worker never accepted this job. return transfer def close(self): diff --git a/tests/gloo_helpers.py b/tests/gloo_helpers.py new file mode 100644 index 0000000..3767ea0 --- /dev/null +++ b/tests/gloo_helpers.py @@ -0,0 +1,27 @@ +"""Retire CPU-only fixture groups after their synchronous callers have stopped.""" + +import gc +from weakref import ref + +from orbitinfer.backend.topology import RankGroup + + +def release_cpu_group_handles(*groups, workers=()): + # All fixture collectives are synchronous, and callers join serve/watchers + # before this boundary. Gloo's shutdown hook is a no-op; its destructor + # joins runloop threads. A quarantined runner can keep these wrappers alive + # after destroy_process_group, so detach only the retired fixture handles. + wrappers = [ + value + for group in groups + for value in vars(group).values() + if isinstance(value, RankGroup) + ] + handles = [ref(group.handle) for group in wrappers] + for worker in workers: + if hasattr(worker, "_wire"): + worker._wire.group = None + for group in wrappers: + object.__setattr__(group, "handle", None) + gc.collect() + assert all(handle() is None for handle in handles), "fixture Gloo owner still live" diff --git a/tests/hardware/cache_storage_cuda.py b/tests/hardware/cache_storage_cuda.py new file mode 100644 index 0000000..b4fec37 --- /dev/null +++ b/tests/hardware/cache_storage_cuda.py @@ -0,0 +1,108 @@ +"""Unrun SM100 gate: public Engine, guarded GPU storage and final-owner containment.""" + +import argparse +import gc +from time import monotonic, sleep +from weakref import ref + +import numpy as np +import torch + +from orbitinfer import ( + CacheGroupLayout, + CacheIdentity, + CacheLayout, + Engine, + ForwardOutput, + MemoryCacheStorage, + RunnerConfig, + SamplingParams, +) +from orbitinfer.backend.cuda.failures import PostAcceptCudaError + + +class Runner: + runner_config = RunnerConfig( + cache_layout=CacheLayout(parent_capacity=8, groups=[CacheGroupLayout(2)]), + continuation_prefill=True, + ) + + def __init__(self, device, fill): + self.arena = torch.full((9, 66), fill, dtype=torch.uint8, device=device) + self.fail = False + + def bind_resources(self, _binding): + pass + + def cache_storage_buffers(self, groups, *, prefix_length): + return tuple( + {"kv": self.arena[slab, 1:65]} for _, rows in groups for slab, _ in rows + ) + + def forward(self, plan): + if self.fail: + raise PostAcceptCudaError("injected uncertain execution") + return ForwardOutput( + token_ids=np.asarray( + [int(plan.token_ids[end - 1]) + 1 for end in plan.token_indptr[1:]], + dtype=np.uint32, + ), + token_indptr=np.arange(plan.num_sequences + 1, dtype=np.uint32), + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda:0") + args = parser.parse_args() + if torch.cuda.get_device_capability(args.device)[0] != 10: + raise RuntimeError("this installed profile requires SM100/Blackwell") + identity = CacheIdentity("gate", "fixed", "raw-v1", "uint8", "tp=0/1", "plain") + store = MemoryCacheStorage(4096) + for fill in (73, 91): + runner = Runner(args.device, fill) + with Engine( + runner, + prefix_cache=True, + prefix_cache_storage=store, + prefix_cache_identity=identity, + prefix_cache_max_bytes=512, + prefix_cache_storage_policy="write_through", + ) as engine: + request = engine.submit( + [1, 2, 3], sampling_params=SamplingParams(max_output_tokens=1) + ) + deadline = monotonic() + 30 + while not request.done: + if monotonic() > deadline: + raise TimeoutError("GPU cache transfer did not finish") + engine.step() + sleep(0.001) + if fill == 91: + assert request.cached_tokens == 2 + host = runner.arena.cpu() + assert (host[:, (0, -1)] == 91).all() + assert (host[:, 1:65] == 73).any() + stats = engine.prefix_storage_stats + assert stats is not None and stats["quarantined_bytes"] == 0 + store.close() + runner = Runner(args.device, 91) + runner.fail = True + weak = ref(runner.arena) + engine = Engine(runner) + request = engine.submit([1], sampling_params=SamplingParams(max_output_tokens=1)) + try: + engine.step() + except PostAcceptCudaError: + pass + else: + raise AssertionError("uncertain execution was reported as success") + assert engine.poisoned and not request.done + del runner, engine, request + gc.collect() + assert weak() is not None + print("SM100 byte/guard and containment controls passed; no full-model claim") + + +if __name__ == "__main__": + main() diff --git a/tests/test_cache_identity.py b/tests/test_cache_identity.py new file mode 100644 index 0000000..fd0c995 --- /dev/null +++ b/tests/test_cache_identity.py @@ -0,0 +1,163 @@ +"""Representation authority and borrowed-byte admission over real CPU views.""" + +from dataclasses import fields, replace +from types import SimpleNamespace + +import pytest +import torch +from test_cache_retirement import HeldStorage +from test_cache_storage import IDENTITY, make_engine + +from orbitinfer import CacheGroupLayout, CacheLayout, SamplingParams +from orbitinfer._cache_storage import _PrefixStorage + + +def binding( + store=None, *, identity=IDENTITY, capacity=8, block=2, limit=16, exporter=None +): + arena = torch.arange(32, dtype=torch.uint8) + + def buffers(groups, *, prefix_length): + return tuple( + {"bytes": arena[row : row + 1]} for _, rows in groups for row, _ in rows + ) + + runner = SimpleNamespace(cache_storage_buffers=exporter or buffers) + return _PrefixStorage( + store or HeldStorage(), + runner, + CacheLayout(parent_capacity=capacity, groups=[CacheGroupLayout(block)]), + identity, + limit, + ) + + +def test_identity_includes_every_authoritative_field_and_geometry(): + key = ([], [1, 2, 3, 4]) + original = binding()._keys(key) + for field in fields(IDENTITY): + changed = replace(IDENTITY, **{field.name: "different"}) + assert set(original).isdisjoint(binding(identity=changed)._keys(key)) + assert set(original).isdisjoint(binding(block=4)._keys(key)) + assert ( + binding(capacity=64)._keys(key) == original + ) # Physical capacity is not format. + assert set(original).isdisjoint(binding()._keys(([1], key[1]))) + + +@pytest.mark.parametrize("invalid", [None, "", 1, "x" * 1025]) +def test_invalid_identity_and_unbounded_budget_rejected_before_io(invalid): + with pytest.raises(ValueError, match="cache identity weights"): + replace(IDENTITY, weights=invalid) + store = HeldStorage() + for identity, limit in [ + (None, 16), + (IDENTITY, 0), + (IDENTITY, -1), + (IDENTITY, 1 << 63), + ]: + with pytest.raises(ValueError, match="prefix_cache"): + binding(store, identity=identity, limit=limit) + assert (store.lookups, store.loads, store.stores) == (0, 0, 0) + + +@pytest.mark.parametrize("failure", [False, True]) +def test_byte_charge_survives_logical_completion_and_failed_receipt(failure): + store = HeldStorage() + owner = binding(store, limit=1) + key, groups = ([], [1, 2]), ((0, ((1, 0),)),) + first = owner.transfer(key, groups, True) + assert owner.stats()["borrowed_bytes"] == 1 + rejected = owner.transfer(key, groups, True) + assert rejected.poll() == 0 and store.loads == 1 + store.transfer.outcome.set_result(False) + assert first.poll() is None and owner.borrowed_bytes == 1 + if failure: + store.transfer.retirement.set_exception(RuntimeError("uncertain fence")) + assert first.poll() is None + assert owner.halted and owner.quarantined_bytes == owner.borrowed_bytes == 1 + assert owner.transfer(key, groups, True).poll() == 0 and store.loads == 1 + else: + store.transfer.retirement.set_result(None) + assert first.poll() == first.poll() == 0 + assert owner.borrowed_bytes == 0 and owner.peak_bytes == 1 + assert owner.transfer(key, groups, True).poll() == 0 and store.loads == 2 + assert owner.borrowed_bytes == 0 + + +@pytest.mark.parametrize( + "case", ["oversize", "overlap", "dtype", "noncontiguous", "too_many"] +) +def test_invalid_exports_never_reach_provider_or_touch_guards(case): + arena = torch.full((16,), 91, dtype=torch.uint8) + store = HeldStorage() + region = {"a": arena[2:6]} + if case == "overlap": + region["b"] = arena[3:7] + elif case == "dtype": + region = {"a": arena.float()} + elif case == "noncontiguous": + region = {"a": arena[::2]} + + def exporter(groups, *, prefix_length): + return (region,) * (2 if case == "too_many" else 1) if groups else () + + owner = binding(store, exporter=exporter, limit=1 if case == "oversize" else 64) + op = owner.transfer(([], [1, 2]), ((0, ((1, 0),)),), True) + assert op.poll() == 0 and store.loads == 0 + assert owner.rejected == 1 and owner.borrowed_bytes == 0 + assert arena.tolist() == [91] * 16 + + +def test_native_storage_admission_consumes_the_byte_budget_and_restores_it(): + store = HeldStorage() + engine, runner, arena = make_engine(store, max_bytes=1) + first = engine.submit( + [1, 2, 3], sampling_params=SamplingParams(max_output_tokens=1) + ) + engine.next_phase() + second = engine.submit( + [4, 5, 6], sampling_params=SamplingParams(max_output_tokens=1) + ) + engine.next_phase() + assert store.loads == 1 and not runner.plans + assert engine.prefix_storage_stats["borrowed_bytes"] == 1 + assert engine.prefix_storage_stats["rejected"] == 1 + assert arena.tolist() == [0] * len(arena) + assert engine.cancel(first) and engine.cancel(second) + store.transfer.outcome.set_result(False) + store.transfer.retirement.set_result(None) + engine.close() + assert engine.prefix_storage_stats["borrowed_bytes"] == 0 + assert engine.metrics_snapshot().cache["storage_pending"] == 0 + + +def test_reference_provider_bounds_queued_jobs_and_bytes_before_enqueue(monkeypatch): + from threading import Event + + from orbitinfer import MemoryCacheStorage + + store = MemoryCacheStorage(4096, max_inflight=1, max_pending_bytes=4) + entered, retire = Event(), Event() + original = store._copy + + def copy(pairs, load): + entered.set() + assert retire.wait(30) + original(pairs, load) + + monkeypatch.setattr(store, "_copy", copy) + source = torch.tensor([1, 2, 3, 4], dtype=torch.uint8) + first = store.store((b"first",), ({"x": source},)) + try: + assert entered.wait(30) + assert not store.store((b"second",), ({"x": source},)).wait(0) + assert (store.pending_count, store.pending_bytes) == (1, 4) + finally: + retire.set() + assert first.wait(30) + assert (store.pending_count, store.pending_bytes) == (0, 0) + assert not store.store( + (b"large",), ({"x": torch.zeros(5, dtype=torch.uint8)},) + ).wait(0) + store.close() diff --git a/tests/test_cache_salt.py b/tests/test_cache_salt.py index 6004f9b..2d88aaf 100644 --- a/tests/test_cache_salt.py +++ b/tests/test_cache_salt.py @@ -88,12 +88,16 @@ def test_storage_namespace_survives_reads_writes_and_engine_recreation(media, po assert complete(engine, cache_salt=salt, **prompt) == 4 -def test_unsalted_storage_keys_are_unchanged_and_disjoint_from_salted_keys(): +def test_storage_keys_bind_identity_and_length_delimited_namespace(): storage = object.__new__(_PrefixStorage) + storage.identity = b"fixture identity" storage.alignment = 2 tokens = [1, 2, 3, 4] encoded = np.asarray(tokens, dtype="