diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 629c9ed..41d61fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,3 +36,12 @@ jobs: run: uv sync --locked --no-default-groups --group test - name: Check source, Rust, Python and installed wheel run: uv run --no-sync python scripts/check.py + - name: Repeat storage and CPU fixture controls + run: | + for iteration in $(seq 1 50); do + echo "CPU owner/fixture repetition $iteration/50" + uv run --no-sync python -m pytest -q tests/test_cache_retirement.py \ + tests/test_distributed_http.py::test_rank_fixture_joins_watcher_before_destroying_process_groups \ + tests/test_deepseek_auxiliary_graphs.py::test_auxiliary_callbacks_bind_model_projections_and_preserve_output_dtypes \ + models/cuda/tests/test_deepseek_v41_projections.py::test_prepared_vocab_head_preserves_last_row_full_logits_and_fp32_output + done diff --git a/README.md b/README.md index 109bec0..d30ab03 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ The complete source import includes CUDA model runners, MLX/Metal runners, Torch reference models, serving tools, benchmarks and tests. The first planned installed target is DeepSeek-V4.1-Flash on SM100. No complete GPU model profile has been qualified for OrbitInfer. See the [plan](docs/completion-plan.md) and -[source import record](docs/source-import.md). +[source import record](docs/source-import.md). Interface changes are recorded in +the [change notes](docs/releases.md). ## Features @@ -33,7 +34,7 @@ Start with the two-page [architecture overview](docs/architecture.md). - [Rust batch scheduler](docs/kernel-boundary.md) - [Unified slab-based cache pool (KV, SWA, Recurrent)](docs/cache-pool.md) - [Pluggable prefix cache storage](docs/cache-storage.md) - - [Hierarchical external KV design](docs/external-kv-design.md) (proposal) + - [Hierarchical external KV design](docs/external-kv-design.md) (adapters planned) - [Speculative decoding (MTP, DSpark)](docs/model-forward-pass.md) - [Zero-overhead overlap scheduling](docs/scheduler.md) - [Engine metrics and Prometheus export](docs/metrics.md) diff --git a/crates/orbitinfer-core/src/engine.rs b/crates/orbitinfer-core/src/engine.rs index d908cd1..c432542 100644 --- a/crates/orbitinfer-core/src/engine.rs +++ b/crates/orbitinfer-core/src/engine.rs @@ -1239,8 +1239,8 @@ impl EngineCore

{ group.append_block_offsets[row] = 0; continue; } - group.append_block_offsets[row] = position % layout.block_tokens().get(); - let logical = position / layout.block_tokens().get(); + group.append_block_offsets[row] = position % layout.block_tokens(); + let logical = position / layout.block_tokens(); let relative = logical .checked_sub(group.block_table_start_indices[row]) .ok_or_else(|| Error::internal("continuation precedes its KV lease"))?; diff --git a/crates/orbitinfer-core/src/planning.rs b/crates/orbitinfer-core/src/planning.rs index 937c0f3..790286f 100644 --- a/crates/orbitinfer-core/src/planning.rs +++ b/crates/orbitinfer-core/src/planning.rs @@ -92,7 +92,7 @@ pub fn schedule_ready_prefix( } else { proposed_end }; - let checkpoint_end = checkpoint_limit / alignment.get() * alignment.get(); + let checkpoint_end = checkpoint_limit / alignment * alignment.get(); if checkpoint_end > request.logical_context && checkpoint_end < proposed_end { count = checkpoint_end - request.logical_context; } diff --git a/crates/orbitinfer-core/src/resources/kv.rs b/crates/orbitinfer-core/src/resources/kv.rs index ba26afc..4677ee2 100644 --- a/crates/orbitinfer-core/src/resources/kv.rs +++ b/crates/orbitinfer-core/src/resources/kv.rs @@ -332,7 +332,7 @@ impl KvLayout { "shared prefix must align to full-history blocks", )); } - let shared_blocks = u64::from(shared_prefix_tokens / group.block_tokens.get()); + let shared_blocks = u64::from(shared_prefix_tokens / group.block_tokens); let extra_copies = u64::try_from(context_lengths.len().saturating_sub(1)) .map_err(|_| overflow())?; blocks -= shared_blocks diff --git a/crates/orbitinfer-core/src/resources/paged_kv.rs b/crates/orbitinfer-core/src/resources/paged_kv.rs index f7b8409..dcaea42 100644 --- a/crates/orbitinfer-core/src/resources/paged_kv.rs +++ b/crates/orbitinfer-core/src/resources/paged_kv.rs @@ -69,7 +69,13 @@ pub struct PrefixCacheMetrics { pub type PrefixStorageGroup = (u32, Vec<(u32, u32)>); pub type StorageOp = Box; +/// A storage job retained with its page loans until physical retirement. pub trait PrefixStorageOperation: fmt::Debug + Send + Sync { + /// Return a matched-prefix length for lookup, or success (one) / failure + /// (zero) for transfer. A terminal transfer result is a release receipt: + /// every access to borrowed backing must have ended, even on failure. + /// `None` retains the job and its loans, including after a failed fence. + /// `wait` permits blocking for the receipt; it never fabricates one. fn poll(&mut self, wait: bool) -> Option; } pub trait PrefixStorage: fmt::Debug + Send + Sync { @@ -307,7 +313,7 @@ impl PagedKvResources { && Some(read) != layout.retention().window_tokens() { let live_first = - (key.len() as u32).saturating_sub(read.get()) / layout.block_tokens().get(); + (key.len() as u32).saturating_sub(read.get()) / layout.block_tokens(); let count = mask.len(); for logical in *first..live_first { mask[logical as usize % count] = false; @@ -469,7 +475,7 @@ impl PagedKvResources { // Wide leases may leave idle spare pages; owner growth never shrinks. let page_count = (group.active_blocks(context).1 as usize).max(table.pages().len()); demands.push(GroupDemand::new(GroupId::new(index as u32), page_count)); - let first = start / group.block_tokens().get(); + let first = start / group.block_tokens(); // A cyclic table is visited at most once, even for long prefills. let count = context .div_ceil(group.block_tokens().get()) @@ -1077,7 +1083,7 @@ impl PagedKvResources { } for position in step.token_start()..post_length { - let logical = position / group.block_tokens().get(); + let logical = position / group.block_tokens(); let page = page_for(group, table.pages(), logical).ok_or_else(|| { Self::invalid(format!( "request {} cache group {group_index} is missing logical block {logical}", @@ -1090,7 +1096,7 @@ impl PagedKvResources { append_offsets.push(if group.is_recurrent() { 0 } else { - position % group.block_tokens().get() + position % group.block_tokens() }); } } diff --git a/crates/orbitinfer-py/src/cache.rs b/crates/orbitinfer-py/src/cache.rs index 4184902..450edd8 100644 --- a/crates/orbitinfer-py/src/cache.rs +++ b/crates/orbitinfer-py/src/cache.rs @@ -31,7 +31,7 @@ fn plan_error(error: KvPlanError) -> PyErr { eq, hash, rename_all = "SCREAMING_SNAKE_CASE", - from_py_object, + skip_from_py_object, module = "orbitinfer._native" )] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -46,7 +46,7 @@ pub(crate) enum CacheRetention { eq, hash, rename_all = "SCREAMING_SNAKE_CASE", - from_py_object, + skip_from_py_object, module = "orbitinfer._native" )] #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] diff --git a/crates/orbitinfer-py/src/lib.rs b/crates/orbitinfer-py/src/lib.rs index fcec2fa..1bb4439 100644 --- a/crates/orbitinfer-py/src/lib.rs +++ b/crates/orbitinfer-py/src/lib.rs @@ -44,6 +44,29 @@ use runner::{ resolve_runner_config, resolve_runner_contract, runner_error_is_post_accept, }; +// PyO3's from_py_object expansion always clones. These enums are Copy; preserve +// its guard errors and inspection hints without suppressing clone_on_copy. +macro_rules! copy_from_py_object { + ($($kind:ty),+ $(,)?) => {$( + impl<'a, 'py> FromPyObject<'a, 'py> for $kind { + type Error = pyo3::pyclass::PyClassGuardError<'a, 'py>; + const INPUT_TYPE: pyo3::inspect::PyStaticExpr = + ::TYPE_HINT; + + fn extract(obj: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(*obj.extract::>()?) + } + } + )+}; +} + +copy_from_py_object!( + CacheRetention, + CacheGroupFamily, + recipe_types::Role, + BatchPhase +); + fn default_sampling_seed(engine_id: u64, request_id: u64) -> i64 { // SplitMix64 gives stable, well-distributed row seeds without adding a // control-plane RNG or making batch composition part of randomness. @@ -137,7 +160,7 @@ impl SchedulerConfig { eq, hash, rename_all = "SCREAMING_SNAKE_CASE", - from_py_object, + skip_from_py_object, module = "orbitinfer._native" )] #[derive( diff --git a/crates/orbitinfer-py/src/recipe_types.rs b/crates/orbitinfer-py/src/recipe_types.rs index c86f858..1271ae3 100644 --- a/crates/orbitinfer-py/src/recipe_types.rs +++ b/crates/orbitinfer-py/src/recipe_types.rs @@ -18,7 +18,7 @@ use serde_json::{Map, Value, json}; /// Some roles are model-specific; recipes validate supported roles. #[non_exhaustive] #[allow(non_camel_case_types, clippy::upper_case_acronyms)] -#[pyclass(frozen, eq, eq_int, from_py_object, module = "orbitinfer._native")] +#[pyclass(frozen, eq, eq_int, skip_from_py_object, module = "orbitinfer._native")] #[derive(Clone, Copy, Debug, PartialEq)] pub(crate) enum Role { SYSTEM, diff --git a/docs/architecture.md b/docs/architecture.md index 5b76436..ca040c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,6 +17,13 @@ enabling an installed profile, prove that actual arena backing, Python owners and registrations survive failed device completion and final-reference drop. The imported quarantine controls alone do not establish that proof. +External storage now reports logical outcome and borrowed-buffer retirement +separately through `CacheTransfer`. The native storage job stays charged until +the binding observes a successful retirement receipt. Failed receipts retain +exact exported views and fence that storage binding, including after final +Engine drop. This proves the Python storage boundary on CPU; it does not close +the remaining native-runner or installed GPU lifetime audit. + ## Runtime owners OrbitInfer separates request and resource ownership from model-specific execution. diff --git a/docs/cache-storage.md b/docs/cache-storage.md index 819d042..b978b37 100644 --- a/docs/cache-storage.md +++ b/docs/cache-storage.md @@ -7,9 +7,9 @@ a storage provider keeps immutable checkpoints outside the custom providers can implement other storage tiers through the same interface. The [hierarchical external KV proposal](external-kv-design.md) specifies identity, -retirement, byte budgets and the planned OrbitKV/Mooncake Store/LMCache adapters. -It is a design for review. This guide describes the currently implemented API; -none of those three adapters is available yet. +byte budgets and the planned OrbitKV/Mooncake Store/LMCache adapters. Independent +operation and retirement receipts are implemented below. The remaining proposal +is not an integration claim; none of those three adapters is available yet. ## Using the DRAM cache @@ -128,15 +128,16 @@ lengths contribute to request cached-token metrics. ## Implementing a provider -The [`CacheStorage` protocol](../orbitinfer/cache_storage.py) has three methods. Each -submits work and returns a `concurrent.futures.Future` promptly, without waiting -for I/O. These are ordinary methods returning futures, not `async def` methods. +The [`CacheStorage` protocol](../orbitinfer/cache_storage.py) has three methods. +Each returns promptly without waiting for I/O. Lookup returns a +`concurrent.futures.Future`; load and store return a `CacheTransfer` containing +two futures. These are ordinary methods, not `async def` methods. | Method | Result | |---|---| | `lookup(keys)` | A tuple of presence booleans in key order. | -| `store(keys, buffers)` | Transfer success, or unknown completion as described below. | -| `load(keys, buffers)` | Transfer success, or unknown completion as described below. | +| `store(keys, buffers)` | `CacheTransfer`: publication outcome and borrowed-source retirement. | +| `load(keys, buffers)` | `CacheTransfer`: whole-load outcome and borrowed-destination retirement. | Keys are opaque `bytes` values. For transfers, `keys` and `buffers` have equal lengths, and `buffers[i]` maps string region names to borrowed, contiguous @@ -158,24 +159,44 @@ providers or partitions for incompatible model or cache representations. ### Transfer completion -A transfer future also defines when OrbitInfer may recycle its buffers: - -- `True` means the transfer succeeded and all accesses to the borrowed buffers - have finished. A successful load must fill every requested region. -- `False`, or a terminal exception, means the operation failed but all buffer - accesses have finished. The Engine can discard a failed restore safely. -- `None` means buffer access may still be outstanding. OrbitInfer retains the owners; - Engine shutdown raises instead of releasing memory that may still be in use. - -Providers must order GPU copies after the work that produced their inputs. A -separate CUDA stream alone does not establish that ordering. Resolve a transfer -with `True`, `False` or an exception only after all borrowed-buffer accesses have -finished. `future.done()` must also return promptly. Cancellation is not proof of -completion. - -Completion covers the borrowed buffers. A provider can manage several tiers by -staging data in its own allocation, then continuing remote I/O against that copy -after the GPU transfer has finished. The provider owns that copy and its lifetime. +`CacheTransfer(outcome, retirement)` separates the two facts: + +- `outcome: Future[bool]` reports success (`True`), a miss/failure (`False`), or an + operation exception. A successful load fills every requested region. A + successful store publishes an immutable object that can be served completely. +- `retirement: Future[None]` succeeds with `None` only when all accesses to the + borrowed buffers have ended. Pending, cancellation, an exception or a non-None + result is not a release receipt. Failed retirement is sticky. + +The Engine consumes an outcome only after successful retirement. A failed +outcome can arrive earlier without releasing pages. A retirement failure fences +new transfers to that storage binding and keeps the operation's exact exported +views, runner and page charges. Close reports failure; dropping the final Engine +reference still retains the unquiesced owners. A provider submission that raises +without returning a receipt is also conservatively unquiesced: it may have +accepted work before raising. Reject capacity before accepting access, then return +a failed outcome with successful retirement when nothing was posted. + +Providers own and complete both futures. Consumers can inspect them but must not +complete them. Cancelling an outcome is not cancellation of DMA. For example, +the DRAM provider acknowledges a cancelled queued operation's retirement only +when its worker proves that no copy was started. `future.done()` alone never +authorizes buffer reuse. Providers must order GPU copies after their producer; +a separate CUDA stream alone does not establish that dependency. + +For standalone use, `transfer.wait(timeout=...)` waits for both receipts within +one timeout budget and returns the bool outcome. It propagates operation errors +only after retirement and raises `CacheRetirementError` for an invalid or failed +retirement receipt. A timeout does not cancel, release or replay the transfer. +The Engine's normal polling uses the same receipts without blocking its scheduler. + +A provider can release the device source after a proven copy into its own charged +staging allocation, then continue remote I/O against that copy. Retirement can +therefore precede publication. The Engine conservatively retains its operation +until both are observed; the provider cannot expose a partial object through +lookup. Its own staging allocation remains retained until the later I/O retires. +`MemoryCacheStorage.close()` raises `CacheRetirementError` after a failed fence; +repeated close does not retry that native cleanup. ## Runner support diff --git a/docs/completion-plan.md b/docs/completion-plan.md index c2fe1d8..d455eb9 100644 --- a/docs/completion-plan.md +++ b/docs/completion-plan.md @@ -8,18 +8,22 @@ deployment tools and authoring skills. Local validation is recorded in the [import evidence](source-import.md). A complete source import does not establish GPU model correctness, performance or production readiness. -Review the [hierarchical external KV design](external-kv-design.md) first: one +Follow the [hierarchical external KV design](external-kv-design.md): one Engine/page allocator, one selected provider and separate outcome/retirement receipts. The proposal covers device residency, provider DRAM/SSD and peer placement, with OrbitKV first, then independent Mooncake Store and LMCache -adapters. It adds no adapter implementation. - -Before provider implementation, make the clean-environment CI baseline pass. -The first import runs exposed newer-Clippy diagnostics and missing test/runtime -dependencies; local passes do not substitute for that gate. - -The next implementation step remains P0: verify final-owner retention through -the existing native/Python storage path, then enforce representation identity +adapters. The first implementation makes load/store outcome and physical +retirement separate receipts through the existing Engine. No provider adapter +has been added. + +Clean-environment validation includes the CPU test dependencies and model-source +paths inherited by subprocess controls. Current Clippy's nonzero-divisor checks +are addressed without changing scheduling arithmetic. Keep this CI baseline +passing before adding providers. + +P0 remains open: the Python storage path now retains exact exported views on +failed retirement and final-reference drop, with forced-interleaving controls. +Next audit native/device execution backing and enforce representation identity and borrowed-byte bounds. P3 begins with the CPU provider contract and OrbitKV DRAM integration; SSD and peer profiles follow separately. GPU integration still needs installed validation. The first deployment target remains DeepSeek on @@ -27,8 +31,8 @@ SM100. Other imported runners are available for separate validation. | Phase | Scope | State | | --- | --- | --- | -| Foundation | Complete runtime, all model packages, tooling, tests and provenance | Source imported; local validation in source-import.md; CI environment/toolchain fixes pending | -| P0 | Final-reference/failed-completion lifetime audit, existing storage integration, real backing, identity and byte bounds | Open; imported quarantine tests alone are insufficient | +| Foundation | Complete runtime, all model packages, tooling, tests and provenance | Source imported; validation in source-import.md; clean CPU gate repairs included | +| P0 | Final-reference/failed-completion lifetime audit, existing storage integration, real backing, identity and byte bounds | Partial: explicit Python storage receipts and backing-retention controls; native/device audit, identity and byte bounds remain open | | P1 | Full-checkpoint offline DeepSeek numerical comparison on SM100 | Unrun; hardware required | | P2 | CUDA graphs, overlap, persistent-state faults and cancellation | Source imported; installed profile unqualified | | P3 | OrbitKV DRAM, then SSD/peer through OrbiTE; separate Mooncake Store and LMCache adapters | Design proposed; not implemented; sequence in external-kv-design.md | diff --git a/docs/external-kv-design.md b/docs/external-kv-design.md index 6963910..b092f4a 100644 --- a/docs/external-kv-design.md +++ b/docs/external-kv-design.md @@ -1,8 +1,9 @@ # Hierarchical external KV cache -Design proposal for review. The current implementation is described in -[prefix cache storage](cache-storage.md); this page specifies its next extension. -No OrbitKV, Mooncake Store or LMCache adapter is implemented by this change. +Design and implementation sequence. The current interface and independent +retirement receipts are described in [prefix cache storage](cache-storage.md). +Identity binding, byte budgets and the provider adapters remain proposed work. +No OrbitKV, Mooncake Store or LMCache adapter is implemented yet. ## Scope @@ -61,7 +62,7 @@ not a cache provider or a global directory. | Existing owner | Keep | Work required before integration | | --- | --- | --- | | `EngineCore` and `PagedKvResources` | Prefix matching, page allocation, storage jobs and commit | Prove final-reference retention of actual backing; add byte accounting for external operations | -| `PrefixStorage` / `PrefixStorageOperation` | Native lookup/transfer/poll boundary | Carry separate outcome and retirement evidence without a second progress loop | +| `PrefixStorage` / `PrefixStorageOperation` | Native lookup/transfer/poll boundary; Python outcome/retirement receipts | Complete the remaining native-runner backing audit without a second progress loop | | `_PrefixStorage` / `CacheStorage` | `lookup`, `load`, `store`; immutable objects | Bind representation identity and validate provider retirement semantics | | `cache_storage_buffers` | Runner's named region views, no export-time copy | Retain arena and registration owners, generation and producer readiness explicitly | | `MemoryCacheStorage` | Bounded DRAM reference provider and existing controls | Audit final-owner and failed-fence paths before declaring the new contract complete | @@ -136,11 +137,12 @@ any copy. An exporter must include all bytes a resumed kernel can read. | Failure and borrowed-buffer retirement proven | Discard private restored pages and recompute, or abandon a backup | | Unknown retirement / failed native cleanup | Quarantine actual owners, keep charges, stop admission to the affected owner; no retry or fabricated completion | -The existing Python future contract encodes these cases as pending, `True`, -`False`/terminal exception, and `None`. A terminal exception currently authorizes -buffer reuse. An adapter must therefore never turn a backend timeout or failed -fence into such an exception while DMA could remain active. P0 must make this -distinction explicit in internal operation state and test it through the binding. +The Python interface now returns `CacheTransfer` with separate outcome and +retirement futures. A logical exception cannot release buffers. The binding +returns a terminal native transfer result only after a successful retirement +receipt; failed retirement is sticky, retains the actual exported views and +fences new storage work. This is tested through the real Engine, including final +reference drop. The wider native-runner and GPU execution audit remains in P0. For load, acquire private destination write leases before accepting physical work, and order copies after source readiness. Make the restored pages visible @@ -220,7 +222,30 @@ LMCache documents a separate MP service with a typed request client. Audit one fixed client/server version before selecting it; do not import a vLLM/SGLang connector as OrbitInfer's scheduler or assume a service reply proves GPU drain. Neither optional adapter introduces a mandatory C++ or Python service into the -Rust Engine. No external backend is a dependency in this design-only change. +Rust Engine. No external backend is a dependency of the core. + +## Correspondence with serving engines + +References checked on 2026-11-15; these are design inputs, not runtime dependencies. +Use the separation of responsibilities without importing another scheduler. + +| Reference boundary | OrbitInfer counterpart | Initial limit | +| --- | --- | --- | +| vLLM `get_num_new_matched_tokens` / `update_state_after_alloc` | Native prefix lookup, then private page reservation | Retain current resident-prefix-first policy | +| vLLM `register_kv_caches` / `start_load_kv` / `save_kv_layer` | Runner exporter and provider load/store | Whole checkpoints initially; no layer-streamed resume | +| vLLM `request_finished` / `get_finished` / `wait_for_save` | Retain page loans until the independent retirement receipt | A request's logical end never releases an active copy | +| SGLang `HiCacheStorage` and registered host pools | One provider owns DRAM/SSD/peer storage; runner retains device backing | No second host-cache manager inside OrbitInfer | +| SGLang `PoolTransfer` and representation/shard configuration | Named checkpoint groups and planned identity binding | No implicit layout conversion or resharding | + +vLLM explicitly lets a connector retain blocks after request completion and +report their completion later. SGLang supplies separate backend operations over +registered pools and named state components. OrbitInfer keeps its existing Rust +`PrefixStorage` owner and Python `CacheStorage` boundary. The independent +`CacheTransfer.retirement` receipt makes buffer-release authority explicit even +when an adapter reports a logical error first. + +Pinned sources: [vLLM connector](https://github.com/vllm-project/vllm/blob/85217ffebbab2905386bac2e38566e09912da25a/vllm/distributed/kv_transfer/kv_connector/v1/base.py) +and [SGLang HiCache storage](https://github.com/sgl-project/sglang/blob/17e4738c4c94310d506328415831724f08ed9614/python/sglang/srt/mem_cache/hicache_storage.py). ## Acceptance and sequence @@ -240,7 +265,8 @@ must force interleavings and inspect backing and budget receipts. 1. **P0: ownership and identity.** Audit the current Rust/Python path, retain actual arena backing across final drops, and add identity and byte bounds. - Prove the contract with `MemoryCacheStorage` and controlled failures. + The Python storage receipt/view-retention controls are implemented. Native + execution backing, identity binding and byte accounting remain open. 2. **P3a: OrbitKV local DRAM.** Add the thin adapter and a CPU contract test. Write the SM100 runner export/producer/readiness gate; run it only on an explicitly authorized installed host. Do not modify OrbitKV from this work. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..5d9e4de --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,13 @@ +# Changes + +## Unreleased + +- External `CacheStorage.load` and `store` now return `CacheTransfer`, with + separate `outcome` and `retirement` futures, instead of a bool/None future. + Update providers to publish a successful retirement receipt only after all + borrowed-buffer accesses have ended. A logical exception alone no longer + releases pages. Use `transfer.wait(timeout=...)` for standalone blocking calls. +- `MemoryCacheStorage.close()` reports a failed device fence through + `CacheRetirementError`; repeated close retains owners without retrying cleanup. +- No OrbitKV, Mooncake Store or LMCache adapter is shipped yet. See the + [external KV design](external-kv-design.md) and [implementation plan](completion-plan.md). diff --git a/models/cuda/tests/test_deepseek_v41_projections.py b/models/cuda/tests/test_deepseek_v41_projections.py index 403c195..5278d1c 100644 --- a/models/cuda/tests/test_deepseek_v41_projections.py +++ b/models/cuda/tests/test_deepseek_v41_projections.py @@ -241,10 +241,12 @@ def prepared(source): def test_prepared_vocab_head_preserves_last_row_full_logits_and_fp32_output(deepgemm): - weight = torch.randn(19, 32).bfloat16().float() + # Last-row and full-logits GEMMs can use different BLAS reduction orders. + # Keep nonuniform signed rows but make their FP32 sums exactly representable. + weight = ((torch.arange(19 * 32) % 17 - 8) / 8).reshape(19, 32) source = SimpleNamespace(weight=weight, tensor_parallel=SimpleNamespace(size=1)) head = PreparedVocabHead(source) - hidden = torch.randn(2, 3, 32).bfloat16() + hidden = ((torch.arange(2 * 3 * 32) % 13 - 6) / 8).reshape(2, 3, 32).bfloat16() expected = F.linear(hidden.float(), weight) assert head.weight.dtype == torch.bfloat16 assert head(hidden).dtype == torch.float32 diff --git a/orbitinfer/__init__.py b/orbitinfer/__init__.py index c08c69a..db906c3 100644 --- a/orbitinfer/__init__.py +++ b/orbitinfer/__init__.py @@ -32,7 +32,12 @@ TokenLogprobs, __version__, ) -from orbitinfer.cache_storage import CacheStorage, MemoryCacheStorage +from orbitinfer.cache_storage import ( + CacheRetirementError, + CacheStorage, + CacheTransfer, + MemoryCacheStorage, +) from orbitinfer.protocols import ( CacheExporter, ManagedRunner, @@ -44,6 +49,8 @@ __all__ = [ "CacheExporter", "CacheStorage", + "CacheTransfer", + "CacheRetirementError", "BatchPhase", "BatchPlan", "CacheGroupFamily", diff --git a/orbitinfer/_cache_storage.py b/orbitinfer/_cache_storage.py index afc7c62..ec5d6ea 100644 --- a/orbitinfer/_cache_storage.py +++ b/orbitinfer/_cache_storage.py @@ -3,10 +3,12 @@ import hashlib import math +from concurrent.futures import Future import numpy as np from orbitinfer._native import CacheGroupFamily +from orbitinfer.cache_storage import CacheTransfer class _Operation: @@ -24,6 +26,46 @@ def poll(self, wait=False): return None if value is None else self.finish(value) +class _TransferOperation: + """Retain actual exported views until the provider proves physical retirement.""" + + def __init__(self, owner, buffers): + self.owner, self.buffers = owner, buffers + self.transfer = None + self.unquiesced = False + + def quarantine(self): + self.unquiesced = True + self.owner.halted = True + + def poll(self, wait=False): + if self.unquiesced or self.transfer is None: + return None + receipt = self.transfer.retirement + if not wait and not receipt.done(): + return None + try: + if receipt.result() is not None: + self.quarantine() + return None + except BaseException: + # A logical failure, cancelled future or failed fence is not a receipt. + self.quarantine() + return None + outcome = self.transfer.outcome + if not wait and not outcome.done(): + return None + try: + # Inspect failure without re-raising a stored exception: its traceback + # would otherwise retain this operation and its retired buffer views. + if outcome.cancelled() or outcome.exception() is not None: + return 0 + result = outcome.result() + return int(result is True) + except BaseException: + return 0 # Only a successful retirement receipt makes failure reusable. + + class _PrefixStorage: def __init__(self, store, runner, layout): self.runner = runner @@ -33,6 +75,7 @@ def __init__(self, store, runner, layout): if buffers((), prefix_length=0): raise ValueError("empty cache export must return no blocks") self.store, self.buffers = store, buffers + self.halted = False if layout is None: raise ValueError("prefix storage requires a managed cache layout") self.groups = layout.groups @@ -57,6 +100,10 @@ def _keys(self, key): return keys def lookup(self, key): + if self.halted: + future = Future() + future.set_result(0) + return _Operation(self, future) bases = self._keys(key) tags = ([b"h"] if self.history else []) + [b"t"] keys = tuple(key + tag for tag in tags for key in bases) @@ -73,6 +120,10 @@ def matched(mask): return _Operation(self, self.store.lookup(keys), matched) def transfer(self, key, groups, load): + if self.halted: + future = Future() + future.set_result(0) # No exporter or provider call was made. + return _Operation(self, future) import torch tensor_type, byte_dtype = torch.Tensor, torch.uint8 @@ -102,5 +153,16 @@ def transfer(self, key, groups, load): keys.append(bases[-1] + b"t") buffers.append(tail) buffers = tuple(buffers) - future = (self.store.load if load else self.store.store)(tuple(keys), buffers) - return _Operation(self, future, buffers=buffers) + operation = _TransferOperation(self, buffers) + try: + transfer = (self.store.load if load else self.store.store)( + tuple(keys), buffers + ) + if not isinstance(transfer, CacheTransfer): + raise TypeError("storage load/store must return CacheTransfer") + operation.transfer = transfer + except BaseException: + # The provider may have accepted a copy before raising. Keep the + # exact views even if it did not return an operation handle. + operation.quarantine() + return operation diff --git a/orbitinfer/cache_storage.py b/orbitinfer/cache_storage.py index d93cf3e..bfd45ec 100644 --- a/orbitinfer/cache_storage.py +++ b/orbitinfer/cache_storage.py @@ -5,7 +5,9 @@ from collections import OrderedDict, defaultdict from collections.abc import Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass from threading import Lock +from time import monotonic from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: @@ -14,16 +16,73 @@ Regions = Mapping[str, "Tensor"] +class CacheRetirementError(RuntimeError): + """Borrowed memory may still be accessed; retain owners and stop admission.""" + + +@dataclass(frozen=True, slots=True) +class CacheTransfer: + """One accepted storage transfer with independent outcome and retirement. + + Providers complete ``outcome`` with a bool or an operation exception. + They complete ``retirement`` successfully with None only after every access + to borrowed buffers has ended. A failed or cancelled retirement future is + not a release receipt. Consumers must not complete these futures themselves. + Cancelling the outcome does not prove retirement or authorize replay. + """ + + outcome: Future[bool] + retirement: Future[None] + + def __post_init__(self): + if not isinstance(self.outcome, Future) or not isinstance( + self.retirement, Future + ): + raise TypeError("cache transfer requires outcome and retirement futures") + if self.outcome is self.retirement: + raise ValueError("outcome and retirement must be separate futures") + + def wait(self, timeout: float | None = None) -> bool: + """Wait for both receipts within one timeout budget. + + A timeout does not cancel I/O or release buffers. Retirement failures + raise CacheRetirementError. Operation exceptions propagate only after + retirement is proven; a false outcome is then safe to discard. + """ + deadline = None if timeout is None else monotonic() + timeout + try: + receipt = self.retirement.result(timeout) + except Exception as error: + if not self.retirement.done(): + raise # A waiting timeout; the provider still owns the operation. + raise CacheRetirementError( + "cache transfer did not prove retirement" + ) from error + if receipt is not None: + raise CacheRetirementError("invalid cache retirement receipt") + remaining = None if deadline is None else max(0, deadline - monotonic()) + outcome = self.outcome.result(remaining) + if type(outcome) is not bool: + raise TypeError("cache transfer outcome must be bool") + return outcome + + class CacheStorage(Protocol): - """Nonblocking submission; terminal transfer results prove buffer retirement.""" + """Nonblocking lookup and transfer submission over immutable byte objects. + + Transfer methods return CacheTransfer promptly. Once submission is attempted, + an exception without a receipt is conservatively unquiesced. Preflight and + resource rejection should return a failed outcome with successful retirement + when no buffer access was accepted. Providers own their internal tiers. + """ def lookup(self, keys: Sequence[bytes]) -> Future[tuple[bool, ...]]: ... def load( self, keys: Sequence[bytes], buffers: Sequence[Regions] - ) -> Future[bool | None]: ... + ) -> CacheTransfer: ... def store( self, keys: Sequence[bytes], buffers: Sequence[Regions] - ) -> Future[bool | None]: ... + ) -> CacheTransfer: ... # A failed device fence cannot authorize reclaiming transfer memory. @@ -208,11 +267,18 @@ def _retire(self): except BaseException: self.halted = True _UNRETIRED.append((self, pending)) - for future, _, _, _, finish, result in pending: + for transfer, _, _, _, finish, result in pending: + if self.halted: + transfer.outcome.set_result(False) + transfer.retirement.set_exception( + CacheRetirementError("cache storage device fence failed") + ) + continue + transfer.retirement.set_result(None) try: - future.set_result(None if self.halted else finish(result)) + transfer.outcome.set_result(finish(result)) except BaseException as error: - future.set_exception(error) # The fence already proved retirement. + transfer.outcome.set_exception(error) # Retirement is already proven. def _retire_task(self): self._retire_queued = False @@ -226,15 +292,20 @@ def _submit(self, action, buffers, finish=lambda value: value, *, defer=False): devices = {r.device for b in buffers for r in b.values() if r.is_cuda} ready = {d: torch.cuda.current_stream(d).record_event() for d in devices} - future = Future() + transfer = CacheTransfer(Future(), Future()) + # Consumers may cancel a queued outcome, but may not cancel the owner's + # retirement accounting. Only run()/the device fence produces its receipt. + transfer.retirement.set_running_or_notify_cancel() def run(): - if not future.set_running_or_notify_cancel(): + if not transfer.outcome.set_running_or_notify_cancel(): + transfer.retirement.set_result(None) # No access was accepted. return if not defer: self._retire() # Eviction must not reuse an earlier read's host bytes. if self.halted: - future.set_result(False) + transfer.outcome.set_result(False) + transfer.retirement.set_result(None) # This operation posted nothing. return try: for device, event in ready.items(): @@ -246,7 +317,7 @@ def run(): result = action() except BaseException: result = False - self._pending.append((future, action, buffers, ready, finish, result)) + self._pending.append((transfer, action, buffers, ready, finish, result)) if not defer: self._retire() else: @@ -259,7 +330,7 @@ def run(): if self.closed: raise RuntimeError("cache storage is closed") self.executor.submit(run) - return future + return transfer def close(self): with self._lock: @@ -268,3 +339,5 @@ def close(self): self._closing = self.executor.submit(self._retire) self._closing.result() self.executor.shutdown(wait=True) + if self.halted: + raise CacheRetirementError("cache storage retains unretired buffers") diff --git a/pyproject.toml b/pyproject.toml index 7308780..543ac96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,10 @@ cuda = [ [dependency-groups] test = [ "pytest>=8,<9", "hypothesis>=6.140,<7", "maturin>=1.9,<2", - "torch==2.13.0", "pillow>=10", "safetensors>=0.5", + "torch==2.13.0", "torchvision==0.28.0", "pillow>=10", "safetensors>=0.5", + # CPU controls import the prepared CUDA modules without launching kernels. + "tilelang==0.1.14; sys_platform == 'linux'", + "tile-kernels==1.0.0; sys_platform == 'linux'", "huggingface-hub>=0.35", "transformers==5.14.1", "tml-renderers==0.2.0", "renderers==0.1.9", "deepseek-recipe==0.2.0", "openai-harmony==0.0.8", diff --git a/scripts/check.py b/scripts/check.py index 9e2e748..4ee313a 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -38,6 +38,12 @@ def main(): # All distributed CPU fixtures use local child processes, not a cluster. cpu_env = os.environ.copy() cpu_env["GLOO_SOCKET_IFNAME"] = "lo0" if sys.platform == "darwin" else "lo" + # The test group deliberately omits the GPU workspace packages. Subprocess + # controls still import their Python source, just as pytest's pythonpath does. + cpu_env["PYTHONPATH"] = os.pathsep.join( + [str(ROOT / "models" / backend) for backend in ("cuda", "torch", "mlx")] + + ([cpu_env["PYTHONPATH"]] if cpu_env.get("PYTHONPATH") else []) + ) run(sys.executable, "-m", "pytest", "-q", "-ra", env=cpu_env) with tempfile.TemporaryDirectory(prefix="orbitinfer-installed-") as temporary: out = Path(temporary) diff --git a/tests/test_cache_retirement.py b/tests/test_cache_retirement.py new file mode 100644 index 0000000..fa8a856 --- /dev/null +++ b/tests/test_cache_retirement.py @@ -0,0 +1,225 @@ +"""Storage outcomes cannot retire a native Engine's actual borrowed views.""" + +import gc +from concurrent.futures import Future, ThreadPoolExecutor, TimeoutError +from threading import Event +from weakref import ref + +import pytest +import torch +from test_cache_storage import make_engine + +from orbitinfer import CacheRetirementError, CacheTransfer, SamplingParams + + +def completed(value): + future = Future() + future.set_result(value) + return future + + +class HeldStorage: + """Only the Engine operation retains the exported tensor objects.""" + + def __init__(self, mode="transfer"): + self.mode = mode + self.transfer = CacheTransfer(Future(), Future()) + self.views = [] + self.loads = self.lookups = self.stores = 0 + + def lookup(self, keys): + self.lookups += 1 + return completed((True,) * len(keys)) + + def load(self, keys, buffers): + self.loads += 1 + self.views = [ref(region) for regions in buffers for region in regions.values()] + if self.mode == "raise": + raise RuntimeError("copy was accepted before submission raised") + if self.mode == "legacy": + # The old bool-future bridge interpreted this exception as a receipt. + future = Future() + future.set_exception(RuntimeError("logical failure before DMA retired")) + return future + return self.transfer + + def store(self, keys, buffers): + self.stores += 1 + return CacheTransfer(completed(False), completed(None)) + + +def pending_load(mode="transfer"): + store = HeldStorage(mode) + engine, runner, arena = make_engine(store, fill=99, limit=1) + request = engine.submit( + [1, 2, 3], sampling_params=SamplingParams(max_output_tokens=1) + ) + engine.next_phase() + assert store.loads == 1 and not runner.plans + assert engine.metrics_snapshot().cache["storage_pending"] == 1 + assert store.views and all(view() is not None for view in store.views) + return engine, runner, arena, request, store + + +@pytest.mark.parametrize("outcome", [True, False, "error", "cancel"]) +def test_logical_outcome_does_not_retire_storage_pages(outcome): + engine, runner, arena, request, store = pending_load() + charged = engine.metrics_snapshot().cache["free_slabs"] + before = arena.clone() + if outcome == "error": + store.transfer.outcome.set_exception(RuntimeError("failed but still writing")) + elif outcome == "cancel": + assert store.transfer.outcome.cancel() + else: + store.transfer.outcome.set_result(outcome) + try: + for _ in range(3): + engine.next_phase() + assert not runner.plans and request.cached_tokens == 0 + assert engine.metrics_snapshot().cache["storage_pending"] == 1 + assert engine.metrics_snapshot().cache["free_slabs"] == charged + assert torch.equal(arena, before) + with pytest.raises(TimeoutError): + store.transfer.wait(0) + assert all(view() is not None for view in store.views) + # A late physical write remains legal until the independent receipt. + touched = [] + for view in store.views: + region = view() + assert region is not None + touched.append(region.data_ptr() - arena.data_ptr()) + region.fill_(73) + del region + store.transfer.retirement.set_result(None) + engine.next_phase() + assert engine.metrics_snapshot().cache["storage_pending"] == 0 + assert request.cached_tokens == (2 if outcome is True else 0) + expected = before.clone() + expected[touched] = 73 + assert torch.equal( + arena, expected + ) # Every byte outside the borrowed views is a guard. + assert all(view() is None for view in store.views) + finally: + if not store.transfer.retirement.done(): + store.transfer.retirement.set_result(None) + engine.close() + + +@pytest.mark.parametrize( + "failure", ["exception", "cancelled", "invalid", "raise", "legacy"] +) +def test_uncertain_retirement_retains_exact_views_after_final_owner_drop(failure): + mode = failure if failure in ("raise", "legacy") else "transfer" + engine, runner, arena, request, store = pending_load(mode) + if mode == "transfer": + store.transfer.outcome.set_result(False) + if failure == "exception": + store.transfer.retirement.set_exception(RuntimeError("failed device fence")) + elif failure == "cancelled": + assert store.transfer.retirement.cancel() + else: + store.transfer.retirement.set_result( + True + ) # Not a valid retirement receipt. + engine.next_phase() + charged = engine.metrics_snapshot().cache["free_slabs"] + assert engine.cancel(request) + engine.clear_prefix_cache() + engine.submit([8, 9, 10], sampling_params=SamplingParams(max_output_tokens=1)) + engine.step() + # No new calls to the uncertain storage owner, including optional backups. + assert (store.lookups, store.loads, store.stores) == (1, 1, 0) + assert engine.metrics_snapshot().cache["storage_pending"] == 1 + assert engine.metrics_snapshot().cache["free_slabs"] <= charged + assert arena.tolist() == [99] * len(arena) + with pytest.raises(RuntimeError): + engine.close() + assert engine.poisoned + retained_runner, retained_arena = ref(runner), ref(arena) + del request, runner, arena, engine + gc.collect() + assert retained_runner() is not None and retained_arena() is not None + assert all(view() is not None for view in store.views) + assert all(view().tolist() == [99] for view in store.views) + + +def test_cancel_and_close_wait_for_storage_retirement(): + engine, _, arena, request, store = pending_load() + assert engine.cancel(request) + store.transfer.outcome.set_result(False) + entered = Event() + + original_result = store.transfer.retirement.result + + def wait_receipt(timeout=None): + entered.set() + return original_result(timeout) + + store.transfer.retirement.result = wait_receipt + with ThreadPoolExecutor(1) as worker: + closing = worker.submit(engine.close) + try: + assert entered.wait(30) + assert not closing.done() + assert all(view() is not None for view in store.views) + assert arena.tolist() == [99] * len(arena) + finally: + store.transfer.retirement.set_result(None) + closing.result(30) + assert engine.closed and not engine.poisoned + assert engine.metrics_snapshot().cache["storage_pending"] == 0 + assert all(view() is None for view in store.views) + + +def test_last_engine_reference_drains_before_releasing_the_arena(): + engine, runner, arena, request, store = pending_load() + store.transfer.outcome.set_result(False) + entered, returned = Event(), Event() + original_result = store.transfer.retirement.result + + def wait_receipt(timeout=None): + entered.set() + return original_result(timeout) + + store.transfer.retirement.result = wait_receipt + retained_runner, retained_arena = ref(runner), ref(arena) + holder = [engine] + del engine, runner, arena, request + + def drop(): + holder.clear() + returned.set() + + with ThreadPoolExecutor(1) as worker: + dropping = worker.submit(drop) + try: + assert entered.wait(30) + assert not returned.is_set() + assert retained_runner() is not None and retained_arena() is not None + assert all(view() is not None for view in store.views) + actual = retained_arena() + assert actual is not None and actual.tolist() == [99] * 33 + del actual + finally: + store.transfer.retirement.set_result(None) + dropping.result(30) + gc.collect() + assert returned.is_set() + assert retained_runner() is None and retained_arena() is None + assert all(view() is None for view in store.views) + + +@pytest.mark.parametrize("invalid", [Future(), None, True]) +def test_transfer_constructor_requires_two_distinct_futures(invalid): + with pytest.raises((TypeError, ValueError)): + CacheTransfer(invalid, invalid) + + +def test_failed_retirement_is_never_reported_as_a_safe_result(): + receipt = Future() + receipt.set_exception(RuntimeError("unconfirmed native cleanup")) + transfer = CacheTransfer(completed(True), receipt) + for _ in range(2): + with pytest.raises(CacheRetirementError): + transfer.wait() diff --git a/tests/test_cache_storage.py b/tests/test_cache_storage.py index 973ebe5..394b1f6 100644 --- a/tests/test_cache_storage.py +++ b/tests/test_cache_storage.py @@ -11,7 +11,11 @@ from test_flat_cache import RecordingCachedRunner from orbitinfer import CacheGroupLayout, CacheLayout, Engine, SamplingParams -from orbitinfer.cache_storage import MemoryCacheStorage +from orbitinfer.cache_storage import ( + CacheRetirementError, + CacheTransfer, + MemoryCacheStorage, +) def make_engine( @@ -99,9 +103,9 @@ def test_memory_bytes_capacity_and_scatter_gather(): known[key] if store.lookup((key,)).result()[0] else bytes(source.tolist()) ) known[key] = expected - assert store.store((key,), ({"x": source[:1], "y": source[1:]},)).result() + assert store.store((key,), ({"x": source[:1], "y": source[1:]},)).wait() for future, target, contents in pending: - assert future.result() and bytes(target.tolist()) == contents + assert future.wait() and bytes(target.tolist()) == contents pending.clear() source.fill_(122) for _ in range(2): @@ -116,20 +120,20 @@ def test_memory_bytes_capacity_and_scatter_gather(): assert store.nbytes <= 10 assert sum(store.lookup((b"a", b"b", b"c")).result()) <= 2 for future, target, contents in pending: - assert future.result() and bytes(target.tolist()) == contents - assert not store.load((key,), ({"different": target},)).result() + assert future.wait() and bytes(target.tolist()) == contents + assert not store.load((key,), ({"different": target},)).wait() store.close() store = MemoryCacheStorage(max_bytes=0) - assert store.store((b"",), ({},)).result() + assert store.store((b"",), ({},)).wait() assert store.lookup((b"",)).result() == (True,) - assert store.load((b"",), ({},)).result() and store.nbytes == 0 + assert store.load((b"",), ({},)).wait() and store.nbytes == 0 store.close() store = MemoryCacheStorage(max_bytes=6) regions = ({"x": torch.tensor([1], dtype=torch.uint8)},) * 3 - assert store.store((b"a", b"b"), regions[:2]).result() - assert store.store((b"a", b"c"), regions[:2]).result() + assert store.store((b"a", b"b"), regions[:2]).wait() + assert store.store((b"a", b"c"), regions[:2]).wait() assert store.lookup((b"a", b"b", b"c")).result() == (True, False, True) - assert not store.store((b"a", b"c", b"d"), regions).result() + assert not store.store((b"a", b"c", b"d"), regions).wait() assert store.lookup((b"a", b"c", b"d")).result() == (True, True, False) store.close() store, other = ( @@ -175,16 +179,16 @@ def _copy(self, pairs, load): store, fence, launch = DelayedCopies(max_bytes=4), Fence(), Event() original = torch.tensor([31, 47], dtype=torch.uint8) - assert store.store((b"a",), ({"x": original},)).result() + assert store.store((b"a",), ({"x": original},)).wait() store.streams["fake"] = fence outputs = [torch.full_like(original, 99) for _ in range(3)] blocker = store.executor.submit(launch.wait) reads = [store.load((b"a",), ({"x": output},)) for output in outputs] - assert reads[-1].cancel() + assert reads[-1].outcome.cancel() reserve = store._reserve def retire_before_reuse(size, protected): - assert all(future.done() for future in reads) + assert all(transfer.retirement.done() for transfer in reads) assert all(torch.equal(output, original) for output in outputs[:2]) return reserve(size, protected) @@ -193,13 +197,15 @@ def retire_before_reuse(size, protected): try: launch.set() assert fence.entered.wait(2) - assert len(fence.copies) == 2 and not any(f.done() for f in reads[:2]) + assert len(fence.copies) == 2 and not any( + t.retirement.done() for t in reads[:2] + ) assert store.lookup((b"a", b"b")).result() == (True, False) - assert not replacement.done() + assert not replacement.retirement.done() finally: fence.ready.set() - assert replacement.result(2) and blocker.result(2) - assert reads[-1].cancelled() and outputs[-1].tolist() == [99, 99] + assert replacement.wait(2) and blocker.result(2) + assert reads[-1].outcome.cancelled() and outputs[-1].tolist() == [99, 99] assert store.lookup((b"a", b"b")).result() == (False, True) launch.clear() @@ -214,11 +220,11 @@ def retire_before_reuse(size, protected): assert fence.entered.wait(2) with pytest.raises(TimeoutError): closing.result(0.02) - assert not any(future.done() for future in closing_reads) + assert not any(t.retirement.done() for t in closing_reads) finally: fence.ready.set() closing.result(2) - assert all(future.result() for future in closing_reads) + assert all(transfer.wait() for transfer in closing_reads) assert all(torch.equal(output, original + 1) for output in outputs[:2]) with pytest.raises(RuntimeError, match="closed"): store.load((b"b",), ({"x": outputs[0]},)) @@ -231,18 +237,21 @@ def synchronize(self): store = MemoryCacheStorage(max_bytes=4) value = torch.tensor([31, 47], dtype=torch.uint8) - assert store.store((b"a",), ({"x": value},)).result() + assert store.store((b"a",), ({"x": value},)).wait() store.streams["fake"] = BrokenFence() destination = torch.empty_like(value) retained = ref(destination) - assert store.load((b"a",), ({"x": destination},)).result() is None + with pytest.raises(CacheRetirementError): + store.load((b"a",), ({"x": destination},)).wait() assert store.halted - assert store.store((b"b",), ({"x": value},)).result() is False + assert store.store((b"b",), ({"x": value},)).wait() is False untouched = torch.full_like(value, 99) - assert store.load((b"a",), ({"x": untouched},)).result() is False + assert store.load((b"a",), ({"x": untouched},)).wait() is False assert untouched.tolist() == [99, 99] and b"a" in store.objects del destination - store.close() + for _ in range(2): + with pytest.raises(CacheRetirementError): + store.close() assert retained() is not None @@ -345,16 +354,27 @@ def test_async_progress_cancellation_and_retirement(delay): method = getattr(store, delay) with ThreadPoolExecutor(3) as workers: - def copy(*args): - if not gate.wait(2): - return False - value = method(*args).result() - return False if delay == "load" else value - def submit(*args): - future = workers.submit(copy, *args) + retirement = Future() + + def copy(): + try: + assert gate.wait(30) + operation = method(*args) + if delay == "lookup": + return operation.result() + operation.retirement.result() + retirement.set_result(None) + value = operation.outcome.result() + return False if delay == "load" else value + except BaseException as error: + if not retirement.done(): + retirement.set_exception(error) + raise + + future = workers.submit(copy) pending.append((future, args[1] if len(args) == 2 else ())) - return future + return future if delay == "lookup" else CacheTransfer(future, retirement) setattr(store, delay, submit) waiting = engine.submit([1, 2, 3], sampling_params=params) diff --git a/tests/test_deepseek_auxiliary_graphs.py b/tests/test_deepseek_auxiliary_graphs.py index 39606ad..ab94d80 100644 --- a/tests/test_deepseek_auxiliary_graphs.py +++ b/tests/test_deepseek_auxiliary_graphs.py @@ -88,7 +88,17 @@ def test_auxiliary_callbacks_bind_model_projections_and_preserve_output_dtypes( ) with torch.inference_mode(), runner._prefill_graphs(): for callback, projection in pairs: - values = torch.randn(1, 3, projection.weight.shape[1]).bfloat16() + # Graph padding changes the CPU GEMM shape. Small dyadic values keep + # every FP32 partial sum exact, so this binding/dtype control can + # retain bitwise equality across BLAS kernels and accumulation orders. + weight = projection.weight + weight.copy_( + ((torch.arange(weight.numel()) % 13 - 6) / 8).reshape_as(weight) + ) + width = weight.shape[1] + values = ( + ((torch.arange(3 * width) % 11 - 5) / 8).reshape(1, 3, width).bfloat16() + ) output = callback(values) assert output.dtype == projection.output_dtype torch.testing.assert_close(output, projection(values), rtol=0, atol=0) diff --git a/tests/test_distributed_http.py b/tests/test_distributed_http.py index 3b30bd0..b1fce89 100644 --- a/tests/test_distributed_http.py +++ b/tests/test_distributed_http.py @@ -121,6 +121,8 @@ def forward_distributed(self, plan, context, *, prepared=None): def _serve_rank(rank, world_size, directory, port, case): patch = pytest.MonkeyPatch() + finished = threading.Event() + stopper = None dist.init_process_group( "gloo", init_method=f"file://{directory}/rendezvous", @@ -283,15 +285,24 @@ def process(tokens): if groups.drivers.local_rank == 0: def stop(): - while not Path(directory, "stop").exists(): - time.sleep(0.01) + while not finished.wait(0.01): + if Path(directory, "stop").exists(): + break server.shutdown() - threading.Thread(target=stop, daemon=True).start() + stopper = threading.Thread(target=stop) + stopper.start() server.serve(port=port) except BaseException as exc: error = str(exc) finally: + # Fault injection can end serve before the parent creates the stop file. + # Do not leave a daemon retaining server/engine/process-group objects + # while the child tears down Gloo or finalizes the Python interpreter. + finished.set() + if stopper is not None: + stopper.join(timeout=30) + assert not stopper.is_alive(), "shutdown watcher did not retire" patch.undo() Path(directory, f"rank-{rank}.json").write_text( json.dumps( @@ -301,6 +312,7 @@ def stop(): "calls": runner.calls, "released": runner.released, "engine_closed": None if engine is None else engine.closed, + "shutdown_joined": stopper is None or not stopper.is_alive(), } ) ) @@ -329,6 +341,57 @@ def _post(base, rank=None, *, content="hello", headers=None, **options): raise +def test_rank_fixture_joins_watcher_before_destroying_process_groups( + tmp_path, monkeypatch +): + """Force serve to return before the parent has published its stop file.""" + threads, receipts = [], [] + original_thread = threading.Thread + + def track_thread(*args, **kwargs): + thread = original_thread(*args, **kwargs) + threads.append(thread) + return thread + + def close_groups(): + assert threads and all(not thread.is_alive() for thread in threads) + receipts.append("groups") + + class EarlyServer: + def __init__(self, *args, **kwargs): + pass + + def serve(self, **kwargs): + assert not (tmp_path / "stop").exists() + raise RuntimeError("forced return before parent stop") + + def shutdown(self): + receipts.append("watcher") + + groups = SimpleNamespace( + is_driver=True, drivers=SimpleNamespace(local_rank=0), close=close_groups + ) + monkeypatch.setattr(threading, "Thread", track_thread) + monkeypatch.setattr(dist, "init_process_group", lambda *args, **kwargs: None) + monkeypatch.setattr(dist, "destroy_process_group", lambda: receipts.append("world")) + monkeypatch.setattr(ParallelGroups, "create", lambda *args, **kwargs: groups) + for name in ("DistributedWorker", "DistributedEngine"): + monkeypatch.setattr( + f"{__name__}.{name}", lambda *args, **kwargs: SimpleNamespace(closed=True) + ) + monkeypatch.setattr(f"{__name__}.HttpServer", EarlyServer) + try: + _serve_rank(0, 2, str(tmp_path), 0, "counts") + assert receipts == ["watcher", "groups", "world"] + assert not (tmp_path / "stop").exists() + finally: + # Also terminates the old daemon when running the negative control. + (tmp_path / "stop").touch() + for thread in threads: + thread.join(timeout=30) + assert not thread.is_alive() + + def _wait_ready(base, directory): deadline = time.monotonic() + 30 while time.monotonic() < deadline: @@ -552,6 +615,7 @@ def test_distributed_http_routes_real_replica_engines(tmp_path, case, world_size for rank in range(world_size) ] assert all(record["closed"] for record in records) + assert all(record["shutdown_joined"] for record in records) if case == "counts": assert all(record["error"] is None for record in records) assert all(not record["calls"] for record in records) diff --git a/tests/test_http_logprobs.py b/tests/test_http_logprobs.py index 4da8b93..ba1e20c 100644 --- a/tests/test_http_logprobs.py +++ b/tests/test_http_logprobs.py @@ -296,7 +296,18 @@ def forward(self, plan): ) results = [] for stream in (False, True): - with running_server(Runner(), recipe) as (_, engine, _, errors, base, _): + # This runner produces tokens without device latency. Keep the whole + # scored response within the bounded event queue even if its HTTP + # consumer is not scheduled until generation finishes. Queue overflow + # and slow-client cancellation have separate lifecycle controls. + with running_server(Runner(), recipe, stream_interval=max(1, len(ids))) as ( + _, + engine, + _, + errors, + base, + _, + ): with request(base, "/v1/" + api, {**body, "stream": stream}) as response: result = collect_scores( response, api=api, stream=stream, partial_tokens=partial_tokens diff --git a/uv.lock b/uv.lock index 7f67163..e8585eb 100644 --- a/uv.lock +++ b/uv.lock @@ -2836,8 +2836,11 @@ test = [ { name = "renderers" }, { name = "ruff" }, { name = "safetensors" }, + { name = "tile-kernels", marker = "sys_platform == 'linux'" }, + { name = "tilelang", marker = "sys_platform == 'linux'" }, { name = "tml-renderers" }, { name = "torch" }, + { name = "torchvision" }, { name = "transformers" }, { name = "ty" }, ] @@ -2895,8 +2898,11 @@ test = [ { name = "renderers", specifier = "==0.1.9" }, { name = "ruff", specifier = ">=0.16,<0.17" }, { name = "safetensors", specifier = ">=0.5" }, + { name = "tile-kernels", marker = "sys_platform == 'linux'", specifier = "==1.0.0" }, + { name = "tilelang", marker = "sys_platform == 'linux'", specifier = "==0.1.14" }, { name = "tml-renderers", specifier = "==0.2.0" }, { name = "torch", specifier = "==2.13.0" }, + { name = "torchvision", specifier = "==0.28.0" }, { name = "transformers", specifier = "==5.14.1" }, { name = "ty", specifier = ">=0.0.90" }, ]