diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f12a29b..a27d5cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: run: | for iteration in $(seq 1 50); do echo "CPU owner/fixture repetition $iteration/50" - uv run --no-sync python -m pytest -q tests/test_cache_retirement.py tests/test_execution_retirement.py tests/test_cache_identity.py \ + uv run --no-sync python -m pytest -q tests/test_cache_retirement.py tests/test_execution_retirement.py tests/test_cache_identity.py tests/test_orbitkv_storage.py \ tests/test_distributed_http.py::test_rank_fixture_joins_watcher_before_destroying_process_groups \ tests/test_distributed_http.py::test_failed_runner_retention_does_not_leave_fixture_gloo_threads_alive \ tests/test_distributed_http.py::test_distributed_http_routes_real_replica_engines[fatal-2] \ diff --git a/AGENTS.md b/AGENTS.md index 106e3e2..cdd95aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,8 @@ Preserve third-party copyright, licenses, notices and the source manifest. - A runner receives a frozen plan. Do not add a second scheduler, page allocator or forwarding coordinator. Storage providers extend the existing interface. - OrbitKV owns external cache identity and policy. OrbiTE owns byte movement. - Their integration is not yet implemented; do not create fake success adapters. + The first OrbitKV adapter uses fixed-block GPU registrations and native receipts. + Do not create fake success adapters or claim unsupported layouts. - Cancel, timeout, logical completion and dropped handles are not physical retirement. Preserve actual backing until its last device/transport borrower retires. Uncertain cleanup retains ownership and stops admission. diff --git a/docs/architecture.md b/docs/architecture.md index a0b5641..872fa4e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -6,9 +6,10 @@ from installed validation; see the [completion plan](completion-plan.md). ## OrbitInfer integration boundaries OrbitInfer owns inference scheduling and device-page leases. A selected external -provider owns cache residency and tier policy. The first planned integration is -OrbitKV, with OrbiTE owning its byte transfers; Mooncake Store and LMCache are -alternative providers. These integrations are not implemented yet. The +provider owns cache residency and tier policy. The first source-level adapter is [OrbitKV local DRAM](orbitkv-dram.md), using its +existing native registration and transfer owners for fixed GPU blocks. Window/ +state mappings and installed validation remain open. Mooncake Store and LMCache +are separate planned providers. The [hierarchical cache proposal](external-kv-design.md) extends the existing storage owners without adding a second scheduler or allocator. diff --git a/docs/cache-storage.md b/docs/cache-storage.md index 405df7e..7ac6c05 100644 --- a/docs/cache-storage.md +++ b/docs/cache-storage.md @@ -8,8 +8,8 @@ custom providers can implement other storage tiers through the same interface. The [hierarchical external KV proposal](external-kv-design.md) specifies identity, byte budgets and the planned OrbitKV/Mooncake Store/LMCache adapters. Identity, -budgets and independent operation/retirement receipts are implemented below. The remaining proposal -is not an integration claim; none of those three adapters is available yet. +budgets and independent operation/retirement receipts are implemented below. The [OrbitKV local DRAM adapter](orbitkv-dram.md) accepts fixed full-history CUDA +blocks. Its installed profile is unqualified; other adapters remain planned. ## Using the DRAM cache diff --git a/docs/completion-plan.md b/docs/completion-plan.md index df2750c..fcee61b 100644 --- a/docs/completion-plan.md +++ b/docs/completion-plan.md @@ -13,8 +13,8 @@ Engine/page allocator, one selected provider and separate outcome/retirement receipts. The proposal covers device residency, provider DRAM/SSD and peer placement, with OrbitKV first, then independent Mooncake Store and LMCache adapters. The first implementation makes load/store outcome and physical -retirement separate receipts through the existing Engine. No provider adapter -has been added. +retirement separate receipts through the existing Engine. A separate OrbitKV adapter maps the existing client to fixed, full-history GPU +blocks in same-host DRAM; its installed profile is unqualified. Clean-environment validation includes the CPU test dependencies and model-source paths inherited by subprocess controls. Current Clippy's nonzero-divisor checks @@ -25,8 +25,7 @@ P0 is implemented at the Rust/Python owner boundary: final drops retain actual backing, uncertain completions and page charges; storage keys bind representation identity and borrowed bytes are bounded until retirement. CPU controls include synchronous failure, asynchronous inspection/harvest failure and a successful -release control. Installed CUDA execution remains unqualified. P3 begins with the CPU provider contract and OrbitKV -DRAM integration; SSD and peer profiles follow separately. GPU integration still +release control. Installed CUDA execution remains unqualified. P3 now has CPU contract controls for the fixed-block OrbitKV DRAM adapter; SSD and peer profiles follow separately. GPU integration still needs installed validation. The first deployment target remains DeepSeek on SM100. Other imported runners are available for separate validation. @@ -36,7 +35,7 @@ SM100. Other imported runners are available for separate validation. | P0 | Final-reference/failed-completion lifetime audit, existing storage integration, real backing, identity and byte bounds | Implemented at source level; real backing, identity, byte and metadata bounds verified on CPU; installed CUDA gate unrun | | P1 | Full-checkpoint offline DeepSeek numerical comparison on SM100 | Unrun; hardware required | | P2 | CUDA graphs, overlap, persistent-state faults and cancellation | Source imported; installed profile unqualified | -| P3 | OrbitKV DRAM, then SSD/peer through OrbiTE; separate Mooncake Store and LMCache adapters | Design proposed; not implemented; sequence in external-kv-design.md | +| P3 | OrbitKV DRAM, then SSD/peer through OrbiTE; separate Mooncake Store and LMCache adapters | Local DRAM adapter implemented for one fixed full-history GPU group; CPU client/CUDA shim controls; installed SM100 gate unrun; SSD/peer/window/state mappings deferred | | P4 | Packaged serving and streaming lifecycle acceptance | Sources imported; model-serving profile unqualified | | P5 | Other CUDA, Torch and MLX runners; measured optimizations | Sources imported; complete-model validation deferred | @@ -65,3 +64,8 @@ OrbiTE integrations remain separate work. `uv run --no-sync python tests/hardware/cache_storage_cuda.py --device cuda:0` checks GPU views, guards, prefix restore and final-owner quarantine on SM100. Written but not run. It does not qualify full-model execution or OrbitKV. + +OrbitKV local DRAM: `uv run --no-sync python tests/hardware/orbitkv_dram.py +--bootstrap-socket /run/orbitkv/orbitkv.sock --metrics-url http://127.0.0.1:9090/metrics +--device cuda:0 --output /tmp/orbitinfer-orbitkv-dram.json`. +Requires a DRAM-only Manager; written but not run. See [layout and route limits](orbitkv-dram.md). diff --git a/docs/external-kv-design.md b/docs/external-kv-design.md index 55a127b..115936f 100644 --- a/docs/external-kv-design.md +++ b/docs/external-kv-design.md @@ -2,7 +2,7 @@ Design and implementation sequence. The current interface and independent retirement receipts are described in [prefix cache storage](cache-storage.md). -Identity binding and borrowed-byte budgets are implemented; provider adapters remain the next step. +Identity binding and borrowed-byte budgets are implemented; the fixed-block OrbitKV local DRAM adapter is implemented at source level; further profiles remain planned. No OrbitKV, Mooncake Store or LMCache adapter is implemented yet. ## Scope @@ -267,7 +267,7 @@ must force interleavings and inspect backing and budget receipts. actual arena backing across final drops, and add identity and byte bounds. The Python storage receipt/view-retention controls are implemented. Native execution backing, identity binding and byte accounting now have CPU controls. Installed device validation is still required. -2. **P3a: OrbitKV local DRAM.** Add the thin adapter and a CPU contract test. +2. **P3a: OrbitKV local DRAM.** The [thin adapter](orbitkv-dram.md) and CPU contract tests cover fixed full-history GPU blocks. Variable window/state mappings remain open. Write the SM100 runner export/producer/readiness gate; run it only on an explicitly authorized installed host. Do not modify OrbitKV from this work. 3. **P3b: OrbitKV SSD and peers.** Separate byte-checked profiles through the diff --git a/docs/orbitkv-dram.md b/docs/orbitkv-dram.md new file mode 100644 index 0000000..516f2a7 --- /dev/null +++ b/docs/orbitkv-dram.md @@ -0,0 +1,111 @@ +# OrbitKV local DRAM + +`OrbitKVCacheStorage` maps fixed GPU KV blocks onto an existing same-host OrbitKV +Manager. OrbitInfer retains the existing prefix matcher, page allocator and +storage operation limits. OrbitKV owns DRAM residency, native registrations, +Publish/Restore and byte movement. The adapter issues no memcpy and does not add +an inference coordinator. No OrbitKV or OrbiTE source changes are required. + +This is **implemented at source level; installed GPU profile unqualified**. +CPU tests drive the real adapter and Engine through an explicit CUDA/client shim, +including byte/guard checks. They are not a GPU or Manager data-path pass. + +## Supported layout + +The first profile is one full-history group, one block per physical slab, and a +single local GPU shard. Each registered layer is a contiguous CUDA `uint8` view +shaped `[physical_block_count, bytes_per_block]`; count includes slab zero. Layers +may have different byte widths but share the block count and device. The runner's +`cache_storage_buffers` must export exactly one whole block for every registered +layer, all at the same physical block index. No allocation or gather is needed +to form a contiguous byte view of an existing fixed block layout. + +Window/state tails, multiple groups, packed slots, partial blocks, resharding and +CPU buffers are rejected. This does **not** yet integrate the full DeepSeek-V4.1 +compressed/window/state layout. Its variable checkpoint slices need a separate +layout mapping; silently padding or concatenating them would change cache +semantics. SSD, peer and multi-shard profiles remain separate work. + +## Binding + +Install an OrbitKV client wheel exposing the native Save/Restore APIs. The source +contract was inspected at +[OrbitKV 93c1c55](https://github.com/OrbitInfra/OrbitKV/tree/93c1c55d795c19cb677f6661763d158a37a059af), +particularly `python/src/client.rs`, `python/src/local_restore.rs` and +`crates/orbitkv-channel/src/client.rs`. The adapter checks required methods before +registration. OrbitInfer does not import OrbitKV or Torch merely on import. + +Configure the Manager for **local DRAM only**. Supply its bootstrap socket and +already allocated runner backing: + +```python +from orbitinfer import CacheIdentity, Engine +from orbitinfer.integrations.orbitkv import OrbitKVCacheStorage + +identity = CacheIdentity( + model="model-id", weights="immutable-weight-revision", format="runner-cache-v1", + dtype="bfloat16", shard="tp=0/1,pp=0/1", semantics="attention-rope-adapter-fingerprint", +) +# These byte views retain the existing GPU allocations. They include null slab 0. +layers = {"k": runner.key_blocks.view(torch.uint8).view(block_count, -1), + "v": runner.value_blocks.view(torch.uint8).view(block_count, -1)} +with OrbitKVCacheStorage( + "/run/orbitkv/orbitkv.sock", identity=identity, layers=layers, + block_tokens=block_tokens, max_pending_bytes=256 * 1024**2, +) as storage: + with Engine(runner, prefix_cache=True, prefix_cache_storage=storage, + prefix_cache_identity=identity, prefix_cache_max_bytes=256 * 1024**2): + ... # Submit and drive requests through the usual Engine API. +``` + +Import `torch` at the caller. Names in `layers` must match the runner's exported +region names. One registration belongs to one Engine. Separate Engines with the +same representation use separate registrations and can share Manager objects. +The namespace includes representation identity, block width and layer names; +physical block capacity and process identity are excluded. Engine keys additionally +include its compiled cache geometry, namespace and token chain. A lookup is only +advisory: load claims a complete lease again before any GPU submission. + +## Ownership + +The adapter has one storage worker, at most 16 admitted operations by default +(configurable 1–64), a required byte limit, and bounded key/region metadata. This +worker bridges the existing blocking native client to `CacheTransfer`; it does +not schedule inference or move bytes. Producer events are recorded on the calling +stream and observed before Publish. Restore also passes that retained stream to +the native client's previous-user fence. + +- `SaveNotSubmittedError` proves no Publish worker was launched and can retire a + failed operation immediately. +- A native `done` result is the client's retirement receipt, including a drained + failure. A pending result, exception or observation timeout is not a receipt. +- Unknown completion retains the client, registration, native handle, exact + views and charges. It fences new submissions and is never replayed. +- Query leases are released if no Restore starts. Once Restore is attempted, + the native client owns consumption; the adapter never guesses whether to + release or replay it after an exception. +- Close the Engine before the provider. Provider close joins its bounded worker, + then unregisters. Failed unregister/close is sticky and is never retried. + Dropping a provider without successful explicit close conservatively retains + its registration; use the context manager. + +`timeout` defaults to 30 seconds (maximum 300) for observing a transfer, not for +releasing memory. No timer fabricates a retirement receipt. The Engine's separate +byte budget remains charged until it consumes the provider's successful receipt. + +## Installed gate + +On an explicitly authorized SM100 host with an installed OrbitKV Manager using +only DRAM, run: + +```bash +uv run --no-sync python tests/hardware/orbitkv_dram.py \ + --bootstrap-socket /run/orbitkv/orbitkv.sock \ + --metrics-url http://127.0.0.1:9090/metrics \ + --device cuda:0 --output /tmp/orbitinfer-orbitkv-dram.json +``` + +The gate checks GPU bytes and guards, prefix restore through public Engine calls, +registration cleanup and route counters. It fails if SSD or peer byte counters +advance. Keep the report outside Git. It is written, not run; no full-model or +hardware qualification is claimed. See the [verification plan](completion-plan.md#installed-validation). diff --git a/docs/releases.md b/docs/releases.md index a82b02d..fe70141 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -19,5 +19,10 @@ releases pages. Use `transfer.wait(timeout=...)` for standalone blocking calls. - `MemoryCacheStorage.close()` reports a failed device fence through `CacheRetirementError`; repeated close retains owners without retrying cleanup. -- No OrbitKV, Mooncake Store or LMCache adapter is shipped yet. See the +- `OrbitKVCacheStorage` adds source-level fixed-block GPU KV to local DRAM + through the existing OrbitKV client. One full-history group is supported; + variable window/state slices and SSD/peers are rejected or outside this profile. + CPU shim controls are not hardware qualification. Explicit provider close is + required; uncertain cleanup and unclosed registrations remain retained. +- Mooncake Store and LMCache adapters remain planned. See the [external KV design](external-kv-design.md) and [implementation plan](completion-plan.md). diff --git a/orbitinfer/_cache_storage.py b/orbitinfer/_cache_storage.py index ee59ac4..89231bb 100644 --- a/orbitinfer/_cache_storage.py +++ b/orbitinfer/_cache_storage.py @@ -130,6 +130,9 @@ def __init__(self, store, runner, layout, identity, max_bytes): if g.family != CacheGroupFamily.STATE and g.window_resident_block_count is None } + validate = getattr(store, "validate_binding", None) + if validate is not None: + validate(identity, layout) def _keys(self, key): namespace, tokens = key diff --git a/orbitinfer/integrations/__init__.py b/orbitinfer/integrations/__init__.py new file mode 100644 index 0000000..617755f --- /dev/null +++ b/orbitinfer/integrations/__init__.py @@ -0,0 +1 @@ +"""Optional external providers. Importing OrbitInfer does not load their runtimes.""" diff --git a/orbitinfer/integrations/orbitkv.py b/orbitinfer/integrations/orbitkv.py new file mode 100644 index 0000000..8ee7560 --- /dev/null +++ b/orbitinfer/integrations/orbitkv.py @@ -0,0 +1,494 @@ +"""Fixed-block CUDA KV to a same-host OrbitKV Manager's DRAM tier. + +The Manager must be configured for DRAM only for this profile. OrbitKV owns +registration, leases and byte movement. This adapter only translates immutable +objects and observes native receipts; it never copies KV bytes itself. +""" + +from __future__ import annotations + +import hashlib +import importlib +import json +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import asdict, dataclass +from threading import Lock +from time import monotonic, sleep +from uuid import uuid4 + +from orbitinfer import ( + CacheGroupFamily, + CacheIdentity, + CacheRetirementError, + CacheTransfer, +) + +# Failed native receipts keep the registration, client and exact borrowed views. +_RETAINED = [] + + +@dataclass +class _Job: + transfer: CacheTransfer + buffers: tuple + charge: int + stream: object + ready: object + handle: object = None + lease: bytes = b"" + + +class OrbitKVCacheStorage: + """One registration of fixed, contiguous CUDA blocks, shared by one Engine. + + `layers` maps exporter region names to contiguous uint8 tensors shaped + [physical_blocks, bytes_per_block]. Include the Engine's reserved null slab. + Only one full-history cache group with one block per slab is supported. + No implicit gather/scatter, state/window conversion, SSD or peer profile is + provided. `timeout` bounds receipt observation, never native lifetime. + + Close the Engine first, then this owner. A failed receipt or unregister is + sticky CacheRetirementError and retains every owner; close never retries it. + """ + + def __init__( + self, + bootstrap_socket, + *, + identity, + layers, + block_tokens, + max_pending_bytes, + max_inflight=16, + timeout=30.0, + ): + import torch + + if not isinstance(identity, CacheIdentity): + raise ValueError("OrbitKV requires explicit CacheIdentity") + if ( + type(block_tokens) is not int + or block_tokens <= 0 + or type(max_inflight) is not int + or not 0 < max_inflight <= 64 + or type(max_pending_bytes) is not int + or not 0 < max_pending_bytes < 1 << 63 + or not 0 < timeout <= 300 + ): + raise ValueError("invalid OrbitKV block, byte, operation or timeout budget") + if not 0 < len(layers) <= 4096: + raise ValueError("OrbitKV requires 1..4096 registered layers") + self.layers = dict(sorted(layers.items())) + for name, tensor in self.layers.items(): + if not isinstance(name, str) or not 0 < len(name.encode()) <= 1024: + raise ValueError("invalid OrbitKV layer name") + if ( + not isinstance(tensor, torch.Tensor) + or not tensor.is_cuda + or tensor.dtype != torch.uint8 + or tensor.ndim != 2 + or not tensor.is_contiguous() + or min(tensor.shape) <= 0 + ): + raise ValueError( + "OrbitKV requires contiguous 2D CUDA uint8 block views" + ) + devices = {tensor.device for tensor in self.layers.values()} + counts = {len(tensor) for tensor in self.layers.values()} + if len(devices) != 1 or len(counts) != 1: + raise ValueError( + "OrbitKV layers must share a device and physical block count" + ) + self.device, self.block_count = devices.pop(), counts.pop() + spans = sorted( + (t.data_ptr(), t.data_ptr() + t.nbytes) for t in self.layers.values() + ) + if any(end > start for (_, end), (start, _) in zip(spans, spans[1:])): + raise ValueError("OrbitKV registered layer views overlap") + self.identity, self.block_tokens = identity, block_tokens + self.max_pending_bytes, self.max_inflight, self.timeout = ( + max_pending_bytes, + max_inflight, + timeout, + ) + self.pending_bytes = self.pending_count = 0 + self._lock, self._close_lock, self._jobs = Lock(), Lock(), {} + self._bound = self._closed = self._retained = False + self._error = None + try: + self.native = importlib.import_module("orbitkv") + except ImportError as error: + raise ImportError( + "OrbitKV storage requires its optional native client wheel" + ) from error + for name in ( + "CacheManagerClient", + "BlockHashes", + "QueryReady", + "QueryCandidates", + "SaveNotSubmittedError", + ): + if not hasattr(self.native, name): + raise RuntimeError( + f"OrbitKV client lacks {name}; see the pinned adapter contract" + ) + for name in ( + "register_context_batch", + "start_save", + "wait_save", + "start_restore", + "wait_restore", + "query_candidates", + "query_prefetch", + "cancel_query", + "release", + "unregister_context", + "close", + ): + if not callable(getattr(self.native.CacheManagerClient, name, None)): + raise RuntimeError( + f"OrbitKV client lacks {name}; see the pinned adapter contract" + ) + self.namespace = ( + "orbitinfer-" + + hashlib.sha256( + json.dumps( + [ + asdict(identity), + block_tokens, + [(n, t.shape[1]) for n, t in self.layers.items()], + ], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + ) + self.instance_id = "orbitinfer-" + uuid4().hex + self.executor = ThreadPoolExecutor(1, thread_name_prefix="orbitinfer-orbitkv") + self.client = self.native.CacheManagerClient(str(bootstrap_socket)) + try: + self.registration = self.client.register_context_batch( + self.instance_id, + self.namespace, + 0, + 0, + 1, + 1, + self.device.index, + list(self.layers), + [self.block_count] * len(self.layers), + [tensor.shape[1] for tensor in self.layers.values()], + [0] * len(self.layers), + [1] * len(self.layers), + "direct", + False, + layer_group_ids=[0] * len(self.layers), + layer_formats=["exact"] * len(self.layers), + tensors=list(self.layers.values()), + ) + except BaseException as error: + self._quarantine(error) + self.executor.shutdown(wait=True) + raise CacheRetirementError( + "OrbitKV registration did not prove cleanup" + ) from error + + def validate_binding(self, identity, layout): + """Check the actual Engine geometry before admitting any cache operation.""" + if self._bound: + raise ValueError("one OrbitKV registration belongs to one Engine") + if identity != self.identity or len(layout.groups) != 1: + raise ValueError("OrbitKV identity/group mismatch") + group = layout.groups[0] + if ( + group.family != CacheGroupFamily.HISTORY + or group.window_resident_block_count is not None + or group.blocks_per_slab != 1 + or group.block_tokens != self.block_tokens + or layout.parent_capacity + 1 != self.block_count + ): + raise ValueError( + "OrbitKV requires one fixed full-history block per physical slab" + ) + self._bound = True + + @staticmethod + def _roots(keys): + keys = tuple(keys) + if not 0 < len(keys) <= 65536 or any( + not isinstance(k, bytes) or len(k) != 33 or k[-1:] not in (b"h", b"t") + for k in keys + ): + raise ValueError("OrbitKV requires bounded Engine checkpoint keys") + return keys, tuple(dict.fromkeys(k[:-1] for k in keys)) + + def _reserve(self, charge): + with self._lock: + if self._closed or self._error: + return False + if ( + self.pending_count >= self.max_inflight + or charge > self.max_pending_bytes - self.pending_bytes + ): + return False + self.pending_count += 1 + self.pending_bytes += charge + return True + + def _refund(self, charge): + with self._lock: + self.pending_count -= 1 + self.pending_bytes -= charge + + def _quarantine(self, error): + with self._lock: + if self._error is None: + self._error = str(error) or type(error).__name__ + if not self._retained: + _RETAINED.append(self) + self._retained = True + + def lookup(self, keys): + keys, roots = self._roots(keys) + result = Future() + if not self._reserve(0): + result.set_result((False,) * len(keys)) + return result + + def run(): + try: + if not result.set_running_or_notify_cancel(): + return + if self._error: + result.set_result((False,) * len(keys)) + return + answer = self.client.query_candidates( + self.instance_id, + self.native.BlockHashes(roots), + uuid4().hex, + ) + hits = ( + set(answer.hit_positions) + if isinstance(answer, self.native.QueryCandidates) + else set() + ) + present = {root for i, root in enumerate(roots) if i in hits} + result.set_result(tuple(key[:-1] in present for key in keys)) + except BaseException as error: + result.set_exception(error) # Advisory lookup cannot access GPU bytes. + finally: + self._refund(0) + + try: + self.executor.submit(run) + except BaseException as error: + self._refund(0) + result.set_exception(error) + return result + + def _plan(self, keys, buffers): + keys, _ = self._roots(keys) + if len(buffers) > 65536 or sum(len(regions) for regions in buffers) > 65536: + raise ValueError("OrbitKV region metadata budget exceeded") + buffers = tuple(dict(regions) for regions in buffers) + if len(keys) != len(buffers) or len(set(keys)) != len(keys): + raise ValueError("OrbitKV key/region count mismatch or duplicate key") + hashes, blocks, charge = [], [], 0 + for key, regions in zip(keys, buffers, strict=True): + if key[-1:] == b"t": + if regions: + raise ValueError( + "OrbitKV variable window/state tails are unsupported" + ) + continue + found = {} + for name, tensor in regions.items(): + if not isinstance(name, str) or len(name.encode()) > 1152: + raise ValueError("OrbitKV region name budget exceeded") + group, logical, layer = name.split("/", 2) + if ( + group != "0" + or not logical.isdecimal() + or layer not in self.layers + or layer in found + ): + raise ValueError("OrbitKV export does not match registered layers") + registered = self.layers[layer] + width = registered.shape[1] + offset = tensor.data_ptr() - registered.data_ptr() + if ( + tensor.device != self.device + or tensor.dtype != registered.dtype + or not tensor.is_contiguous() + or tensor.nbytes != width + or offset < 0 + or offset % width + or offset + width > registered.nbytes + ): + raise ValueError("OrbitKV export is not one exact registered block") + found[layer] = offset // width + charge += width + if charge > self.max_pending_bytes: + raise ValueError("OrbitKV borrowed byte budget exceeded") + if found.keys() != self.layers.keys() or len(set(found.values())) != 1: + raise ValueError("OrbitKV requires complete layers on the same block") + hashes.append(key[:-1]) + blocks.append(next(iter(found.values()))) + if not hashes or len(set(blocks)) != len(blocks): + raise ValueError("OrbitKV requires non-overlapping full-history blocks") + return buffers, hashes, blocks, charge + + def store(self, keys, buffers): + return self._submit(keys, buffers, load=False) + + def load(self, keys, buffers): + return self._submit(keys, buffers, load=True) + + def _finish(self, job, success): + with self._lock: + self._jobs.pop(id(job), None) + self._refund(job.charge) + job.transfer.retirement.set_result(None) + if not job.transfer.outcome.cancelled(): + job.transfer.outcome.set_result(success) + + def _submit(self, keys, buffers, *, load): + import torch + + transfer = CacheTransfer(Future(), Future()) + transfer.retirement.set_running_or_notify_cancel() + try: + buffers, hashes, blocks, charge = self._plan(keys, buffers) + except (ValueError, TypeError, AttributeError): + transfer.outcome.set_result(False) + transfer.retirement.set_result(None) + return transfer + if not self._reserve(charge): + transfer.outcome.set_result(False) + transfer.retirement.set_result(None) + return transfer + job = _Job(transfer, buffers, charge, None, None) + with self._lock: + self._jobs[id(job)] = job + try: + job.stream = torch.cuda.current_stream(self.device) + job.ready = job.stream.record_event() + self.executor.submit(self._run, job, hashes, blocks, load) + except BaseException as error: + self._fail(job, error) + return transfer + + def _fail(self, job, error): + self._quarantine(error) + if not job.transfer.outcome.done(): + job.transfer.outcome.set_result(False) + job.transfer.retirement.set_exception( + CacheRetirementError( + "OrbitKV did not prove retirement; registration, views and charges retained" + ) + ) + + def _run(self, job, hashes, blocks, load): + try: + if not job.transfer.outcome.set_running_or_notify_cancel(): + self._finish(job, False) + return + if self._error: + self._finish(job, False) # This queued job posted nothing. + return + deadline = monotonic() + self.timeout + while not job.ready.query(): + if monotonic() >= deadline: + raise TimeoutError("producer event did not complete") + sleep(0.001) + if load: + request_id = uuid4().hex + while True: + answer = self.client.query_prefetch( + self.instance_id, + self.native.BlockHashes(hashes), + request_id, + ) + if isinstance(answer, self.native.QueryReady): + break + if monotonic() >= deadline: + self.client.cancel_query(self.instance_id, request_id) + self._finish(job, False) # Restore was never called. + return + sleep(0.001) + job.lease = answer.lease + if answer.num_hit_blocks != len(blocks) or set( + answer.hit_positions + ) != set(range(len(blocks))): + if job.lease: + self.client.release(job.lease) + self._finish(job, False) + return + job.handle = self.client.start_restore( + self.instance_id, + 0, + self.device.index, + [list(self.layers)], + [(job.lease, [blocks])], + ready_stream=job.stream.cuda_stream, + ) + status = self.client.wait_restore( + job.handle, timeout=max(0, deadline - monotonic()) + ) + else: + try: + job.handle = self.client.start_save( + self.registration, + [(name, blocks, hashes) for name in self.layers], + ) + except self.native.SaveNotSubmittedError: + self._finish(job, False) + return + status = self.client.wait_save( + job.handle, timeout=max(0, deadline - monotonic()) + ) + if type(status.done) is not bool or type(status.success) is not bool: + raise CacheRetirementError("invalid native retirement status") + if not status.done: + raise CacheRetirementError( + status.message or "native operation remains pending" + ) + self._finish(job, bool(status.success)) + except BaseException as error: + self._fail(job, error) + + def close(self): + with self._close_lock: + self._close() + + def _close(self): + with self._lock: + self._closed = True + self.executor.shutdown(wait=True) + if self._error: + raise CacheRetirementError(self._error) + if not hasattr(self, "registration"): + return + try: + success, message = self.client.unregister_context(self.instance_id) + if not success: + raise CacheRetirementError(message or "OrbitKV unregister failed") + self.client.close() + del self.registration + except BaseException as error: + self._quarantine(error) + raise CacheRetirementError( + "OrbitKV cleanup uncertain; never retried" + ) from error + + def __del__(self): + # Dropping a configured provider is not permission to deregister. The + # explicit close path joins its bounded executor and receives cleanup. + if hasattr(self, "registration"): + self._quarantine("OrbitKV provider dropped without successful close") + + def __enter__(self): + return self + + def __exit__(self, *_error): + self.close() diff --git a/scripts/check.py b/scripts/check.py index 4ee313a..e2b8988 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -60,7 +60,8 @@ def main(): "-I", "-c", "import sys, orbitinfer; " - "imported = {'torch', 'resin', 'flashinfer', 'mlx'} & sys.modules.keys(); " + "from orbitinfer.integrations.orbitkv import OrbitKVCacheStorage; " + "imported = {'torch', 'orbitkv', 'resin', 'flashinfer', 'mlx'} & sys.modules.keys(); " "wrong = orbitinfer.Engine.__module__ != 'orbitinfer._native'; " "sys.exit(f'unexpected import: {imported}, binding={wrong}' if imported or wrong else 0)", cwd=out, diff --git a/tests/hardware/orbitkv_dram.py b/tests/hardware/orbitkv_dram.py new file mode 100644 index 0000000..c54ae7e --- /dev/null +++ b/tests/hardware/orbitkv_dram.py @@ -0,0 +1,159 @@ +"""Unrun SM100 GPU KV / local OrbitKV DRAM gate. Start a DRAM-only Manager first.""" + +import argparse +import json +from importlib.metadata import version +from pathlib import Path +from time import monotonic, sleep +from urllib.request import urlopen + +import numpy as np +import torch + +from orbitinfer import ( + CacheGroupLayout, + CacheIdentity, + CacheLayout, + Engine, + ForwardOutput, + RunnerConfig, + SamplingParams, +) +from orbitinfer.integrations.orbitkv import OrbitKVCacheStorage + + +class Runner: + runner_config = RunnerConfig( + cache_layout=CacheLayout(parent_capacity=8, groups=[CacheGroupLayout(2)]), + continuation_prefill=True, + ) + + def __init__(self, device, fill, width): + self.layers = { + name: torch.full((9, width), fill, device=device, dtype=torch.uint8) + for name in ("k", "v") + } + + def bind_resources(self, _binding): + pass + + def cache_storage_buffers(self, groups, *, prefix_length): + return tuple( + {name: tensor[slab] for name, tensor in self.layers.items()} + for _, rows in groups + for slab, _ in rows + ) + + def forward(self, plan): + return ForwardOutput( + token_ids=np.asarray( + [int(plan.token_ids[end - 1]) + 1 for end in plan.token_indptr[1:]], + dtype=np.uint32, + ), + token_indptr=np.arange(plan.num_sequences + 1, dtype=np.uint32), + ) + + +def metrics(url): + result = {} + with urlopen(url, timeout=10) as response: + for line in response.read(8 << 20).decode().splitlines(): + if line and not line.startswith("#"): + name, value = line.rsplit(" ", 1) + name = name.split("{", 1)[0] + result[name] = result.get(name, 0) + float(value) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--bootstrap-socket", required=True) + parser.add_argument("--metrics-url", required=True) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + if torch.cuda.get_device_capability(args.device)[0] != 10: + raise RuntimeError("this installed profile requires SM100/Blackwell") + before = metrics(args.metrics_url) + checks = [] + for width in (4096, 65536): + identity = CacheIdentity( + "installed-gate", + "fixed", + f"bytes-{width}", + "uint8", + "tp=0/1,pp=0/1", + "full-history", + ) + for fill in (73, 91): + runner = Runner(args.device, fill, width) + with OrbitKVCacheStorage( + args.bootstrap_socket, + identity=identity, + layers=runner.layers, + block_tokens=2, + max_pending_bytes=8 * width, + ) as storage: + with Engine( + runner, + prefix_cache=True, + prefix_cache_storage=storage, + prefix_cache_identity=identity, + prefix_cache_max_bytes=8 * width, + prefix_cache_storage_policy="write_through", + ) as engine: + request = engine.submit( + [1, 2, 3], sampling_params=SamplingParams(max_output_tokens=1) + ) + deadline = monotonic() + 60 + while not request.done: + if monotonic() > deadline: + raise TimeoutError( + "OrbitKV Engine transfer did not complete" + ) + engine.step() + sleep(0.001) + if request.take_result() != [4]: + raise AssertionError("wrong continuation output") + if fill == 91: + if request.cached_tokens != 2: + raise AssertionError("no OrbitKV prefix restore") + for tensor in runner.layers.values(): + actual = tensor.cpu() + if not ( + (actual[1] == 73).all() + and (actual[0] == 91).all() + and (actual[2:] == 91).all() + ): + raise AssertionError("payload or guard mismatch") + if storage.pending_count or storage.pending_bytes: + raise AssertionError("drained Engine still has a charged transfer") + checks.append(dict(width=width, fill=fill, bytes_verified=True)) + after = metrics(args.metrics_url) + for name in ("orbitkv_save_bytes_total", "orbitkv_load_bytes_total"): + if name not in before or name not in after or after[name] <= before[name]: + raise AssertionError(f"missing native route evidence: {name}") + for name in set(before) | set(after): + if ("ssd" in name or "remote" in name) and "bytes" in name: + if after.get(name, 0) > before.get(name, 0): + raise AssertionError(f"non-DRAM route observed: {name}") + args.output.write_text( + json.dumps( + dict( + orbitinfer=version("orbitinfer"), + orbitkv=version("orbitkv"), + torch=torch.__version__, + device=torch.cuda.get_device_name(args.device), + checks=checks, + metrics_before=before, + metrics_after=after, + scope="GPU KV/local DRAM; no full model or SSD/peer claim", + ), + indent=2, + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_orbitkv_storage.py b/tests/test_orbitkv_storage.py new file mode 100644 index 0000000..0a5eef6 --- /dev/null +++ b/tests/test_orbitkv_storage.py @@ -0,0 +1,498 @@ +"""CPU client/CUDA shims exercise the production adapter; not a GPU profile.""" + +import gc +import sys +from concurrent.futures import Future +from threading import Event +from types import ModuleType, SimpleNamespace +from weakref import ref + +import pytest +import torch +from test_cache_storage import IDENTITY + +from orbitinfer import CacheGroupLayout, CacheLayout +from orbitinfer.cache_storage import CacheRetirementError +from orbitinfer.integrations.orbitkv import OrbitKVCacheStorage + + +class CudaView(torch.Tensor): + @property + def device(self): # ty: ignore[invalid-property-type-override] -- CPU CUDA-claim shim + return torch.device("cuda:0") + + @property + def is_cuda(self): # ty: ignore[invalid-property-type-override] -- CPU CUDA-claim shim + return True + + +class QueryCandidates: + def __init__(self, positions): + self.hit_positions = positions + + +class QueryReady(QueryCandidates): + def __init__(self, lease, positions): + super().__init__(positions) + self.lease, self.num_hit_blocks = lease, len(positions) + + +class SaveNotSubmittedError(RuntimeError): + pass + + +@pytest.fixture +def native(monkeypatch): + data, calls = {}, [] + gate = Future() + gate.set_result(SimpleNamespace(done=True, success=True, message="")) + state = SimpleNamespace( + gate=gate, entered=Event(), ready=True, mode="ok", instances=[] + ) + + class Client: + def __init__(self, socket): + self.socket = socket + self.layers = {} + self.leases = {} + state.instances.append(ref(self)) + + def register_context_batch( + self, + instance, + namespace, + tp, + pp, + tp_size, + world, + device, + names, + counts, + widths, + strides, + segments, + backend, + page_first, + *, + tensors, + **metadata, + ): + calls.append( + ( + "register", + namespace, + tp, + pp, + tp_size, + world, + device, + backend, + page_first, + ) + ) + assert strides == [0] * len(names) and segments == [1] * len(names) + assert counts == [len(t) for t in tensors] and widths == [ + t.shape[1] for t in tensors + ] + assert metadata == dict( + layer_group_ids=[0] * len(names), layer_formats=["exact"] * len(names) + ) + self.layers = dict(zip(names, tensors, strict=True)) + self.namespace = namespace + if state.mode == "register_fail": + raise RuntimeError("uncertain registration") + return object() + + def query_candidates(self, instance, hashes, req): + calls.append(("lookup", tuple(hashes))) + return QueryCandidates( + [ + i + for i, h in enumerate(hashes) + if all((self.namespace, name, h) in data for name in self.layers) + ] + ) + + def query_prefetch(self, instance, hashes, req): + answer = self.query_candidates(instance, hashes, req) + self.leases[b"lease"] = hashes + return QueryReady(b"lease", answer.hit_positions) + + def start_save(self, registration, saves): + calls.append(("save", saves)) + if state.mode == "rejected": + raise SaveNotSubmittedError("never submitted") + if state.mode == "raise": + raise RuntimeError("accepted before error") + state.entered.set() + return ("save", saves) + + def start_restore(self, instance, tp, device, groups, loads, *, ready_stream): + calls.append(("restore", ready_stream, groups, loads)) + assert ready_stream == 73 and groups == [list(self.layers)] + if state.mode == "restore_raise": + self.leases.pop(loads[0][0]) # Native attempt consumed the lease. + raise RuntimeError("restore accepted before error") + state.entered.set() + return ("load", loads) + + def wait_save(self, handle, *, timeout): + return self.wait(handle, timeout) + + def wait_restore(self, handle, *, timeout): + return self.wait(handle, timeout) + + def wait(self, handle, timeout): + status = state.gate.result(timeout=timeout) + if status.done and status.success: + kind, tasks = handle + if kind == "save": + for layer, blocks, hashes in tasks: + for block, h in zip(blocks, hashes, strict=True): + data[self.namespace, layer, h] = self.layers[layer][ + block + ].clone() + else: + for lease, (blocks,) in tasks: + for block, h in zip( + blocks, self.leases.pop(lease), strict=True + ): + for layer, tensor in self.layers.items(): + tensor[block].copy_(data[self.namespace, layer, h]) + return status + + def release(self, lease): + calls.append(("release", lease)) + self.leases.pop(lease) + + def cancel_query(self, instance, request): + calls.append(("cancel_query", request)) + + def unregister_context(self, instance): + calls.append(("unregister", instance)) + if state.mode == "unregister_fail": + return False, "uncertain unregister" + self.layers.clear() + return True, "" + + def close(self): + calls.append(("close",)) + + module = ModuleType("orbitkv") + for name, value in dict( + CacheManagerClient=Client, + QueryReady=QueryReady, + QueryCandidates=QueryCandidates, + BlockHashes=tuple, + SaveNotSubmittedError=SaveNotSubmittedError, + ).items(): + setattr(module, name, value) + monkeypatch.setitem(sys.modules, "orbitkv", module) + stream = SimpleNamespace( + cuda_stream=73, record_event=lambda: SimpleNamespace(query=lambda: state.ready) + ) + monkeypatch.setattr(torch.cuda, "current_stream", lambda device: stream) + return state, data, calls + + +def storage(*, identity=IDENTITY, max_pending_bytes=128, bind=True, **kwargs): + layers = { + name: torch.full((9, 8), 91, dtype=torch.uint8).as_subclass(CudaView) + for name in ("k", "v") + } + owner = OrbitKVCacheStorage( + "/fixture.sock", + identity=identity, + layers=layers, + block_tokens=2, + max_pending_bytes=max_pending_bytes, + **kwargs, + ) + if bind: + owner.validate_binding( + identity, CacheLayout(parent_capacity=8, groups=[CacheGroupLayout(2)]) + ) + return owner + + +def vector(owner, blocks=(1, 2)): + keys = tuple(bytes([i]) * 32 + b"h" for i in range(len(blocks))) + buffers = tuple( + {f"0/{i}/{name}": tensor[block] for name, tensor in owner.layers.items()} + for i, block in enumerate(blocks) + ) + return keys + (keys[-1][:-1] + b"t",), buffers + ({},) + + +def test_gpu_block_contract_roundtrip_and_guards_without_adapter_byte_copies(native): + source, target = storage(), storage() + for tensor in source.layers.values(): + tensor[1:3].fill_(73) + keys, buffers = vector(source) + assert source.store(keys, buffers).wait(30) + assert target.lookup(keys).result(30) == (True, True, True) + _, targets = vector(target, (5, 6)) + assert target.load(keys, targets).wait(30) + for tensor in target.layers.values(): + actual = tensor.as_subclass(torch.Tensor) + assert (actual[5:7] == 73).all() + assert (actual[:5] == 91).all() and (actual[7:] == 91).all() + assert source.pending_bytes == target.pending_bytes == 0 + source.close() + target.close() + assert [row[0] for row in native[2]].count("save") == 1 + assert [row[0] for row in native[2]].count("restore") == 1 + + +@pytest.mark.parametrize( + "mode", ["pending", "raise", "unregister_fail", "register_fail"] +) +def test_unknown_receipt_or_cleanup_is_sticky_and_keeps_actual_owners(native, mode): + state, _, calls = native + if mode == "register_fail": + state.mode = mode + with pytest.raises(CacheRetirementError): + storage() + assert state.instances[-1]() is not None + assert not any(c[0] in ("close", "unregister") for c in calls) + return + owner = storage() + weak = ref(owner.layers["k"]) + state.mode = mode + if mode == "pending": + state.gate = Future() + state.gate.set_result( + SimpleNamespace(done=False, success=False, message="retained") + ) + if mode != "unregister_fail": + transfer = owner.store(*vector(owner)) + with pytest.raises(CacheRetirementError): + transfer.wait(30) + assert owner.pending_bytes == 32 and owner.pending_count == 1 + assert not owner.store(*vector(owner)).wait(30) + for _ in range(2): + with pytest.raises(CacheRetirementError): + owner.close() + assert [c[0] for c in calls].count("unregister") == int(mode == "unregister_fail") + assert not any(c[0] == "close" for c in calls) + del owner + gc.collect() + assert weak() is not None + + +def test_pending_save_holds_views_and_budget_until_native_drain(native): + state, _, calls = native + owner = storage(max_pending_bytes=32) + state.gate = Future() + keys, views = vector(owner) + weak = ref(views[0]["0/0/k"]) + op = owner.store(keys, views) + assert state.entered.wait(30) + del views + gc.collect() + assert weak() is not None and not op.retirement.done() + assert not owner.store(*vector(owner)).wait(0) + assert owner.pending_bytes == 32 + state.gate.set_result(SimpleNamespace(done=True, success=True, message="")) + assert op.wait(30) + owner.close() + gc.collect() + assert weak() is None and owner.pending_bytes == 0 + assert [c[0] for c in calls].count("save") == 1 + + +def test_no_access_rejection_and_invalid_layout_never_submit(native): + state, _, calls = native + owner = storage() + state.mode = "rejected" + assert not owner.store(*vector(owner)).wait(30) + assert owner.pending_bytes == 0 + before = len(calls) + keys, buffers = vector(owner) + buffers[0]["0/0/k"] = owner.layers["k"][1, :4] + assert not owner.load(keys, buffers).wait(0) + assert len(calls) == before and owner.pending_bytes == 0 + with pytest.raises(ValueError, match="one Engine"): + owner.validate_binding( + IDENTITY, CacheLayout(parent_capacity=8, groups=[CacheGroupLayout(2)]) + ) + owner.close() + + +def test_queued_cancel_is_no_access_but_running_cancel_is_not_retirement(native): + state, _, calls = native + owner = storage() + state.gate = Future() + first = owner.store(*vector(owner)) + assert state.entered.wait(30) + assert not first.outcome.cancel() and not first.retirement.done() + second = owner.store(*vector(owner)) + assert second.outcome.cancel() and not second.retirement.done() + assert owner.pending_count == 2 and owner.pending_bytes == 64 + state.gate.set_result(SimpleNamespace(done=True, success=True, message="")) + assert first.wait(30) + second.retirement.result(30) + owner.close() + assert owner.pending_bytes == owner.pending_count == 0 + assert [c[0] for c in calls].count("save") == 1 + + +def test_close_joins_pending_copy_before_deregistering(native): + from concurrent.futures import ThreadPoolExecutor + + state, _, calls = native + owner = storage() + state.gate = Future() + operation = owner.store(*vector(owner)) + assert state.entered.wait(30) + entered = Event() + shutdown = owner.executor.shutdown + + def joining(*args, **kwargs): + entered.set() + return shutdown(*args, **kwargs) + + owner.executor.shutdown = joining + with ThreadPoolExecutor(1) as worker: + closed = worker.submit(owner.close) + assert entered.wait(30) + assert not closed.done() and not operation.retirement.done() + assert not any(c[0] == "unregister" for c in calls) + state.gate.set_result(SimpleNamespace(done=True, success=True, message="")) + closed.result(30) + assert operation.wait(0) and owner.pending_bytes == 0 + owner.close() # Successful close is idempotent, failed close is sticky. + assert [c[0] for c in calls].count("unregister") == 1 + + +def test_missing_client_method_and_cpu_tensor_rejected_before_registration( + native, monkeypatch +): + from orbitinfer.integrations.orbitkv import OrbitKVCacheStorage + + with pytest.raises(ValueError, match="CUDA"): + OrbitKVCacheStorage( + "/fixture", + identity=IDENTITY, + layers={"x": torch.zeros((2, 8), dtype=torch.uint8)}, + block_tokens=2, + max_pending_bytes=16, + ) + assert not native[2] + monkeypatch.setattr(sys.modules["orbitkv"].CacheManagerClient, "wait_restore", None) + with pytest.raises(RuntimeError, match="wait_restore"): + storage() + assert not native[2] + + +def test_identity_namespace_cannot_load_other_weights(native): + from dataclasses import replace + + source = storage() + assert source.store(*vector(source)).wait(30) + other = storage(identity=replace(IDENTITY, weights="another-checkpoint")) + keys, buffers = vector(other) + assert other.lookup(keys).result(30) == (False, False, False) + assert not other.load(keys, buffers).wait(30) + assert all((t.as_subclass(torch.Tensor) == 91).all() for t in other.layers.values()) + source.close() + other.close() + + +@pytest.mark.parametrize("length", [3, 5, 7]) +def test_real_engine_restores_only_after_the_native_client_receipt( + native, monkeypatch, length +): + from time import monotonic, sleep + + from test_flat_cache import RecordingCachedRunner + + from orbitinfer import Engine, SamplingParams + + for fill in (73, 91): + owner = storage(bind=False) + runner = RecordingCachedRunner( + CacheLayout(parent_capacity=8, groups=[CacheGroupLayout(2)]) + ) + for tensor in owner.layers.values(): + tensor.fill_(fill) + + def export(groups, *, prefix_length): + return tuple( + {name: tensor[slab] for name, tensor in owner.layers.items()} + for _, rows in groups + for slab, _ in rows + ) + + monkeypatch.setattr(runner, "cache_storage_buffers", export, raising=False) + with Engine( + runner, + prefix_cache=True, + prefix_cache_storage=owner, + prefix_cache_identity=IDENTITY, + prefix_cache_max_bytes=128, + prefix_cache_storage_policy="write_through", + ) as engine: + request = engine.submit( + list(range(1, length + 1)), + sampling_params=SamplingParams(max_output_tokens=1), + ) + deadline = monotonic() + 30 + while not request.done: + assert monotonic() < deadline + engine.step() + sleep(0.001) + assert request.cached_tokens == (length - 1 if fill == 91 else 0) + assert request.take_result() == [length + 1] + if fill == 91: + for tensor in owner.layers.values(): + actual = tensor.as_subclass(torch.Tensor) + end = 1 + length // 2 + assert (actual[1:end] == 73).all() and (actual[0] == 91).all() + assert (actual[end:] == 91).all() + owner.close() + + +@pytest.mark.parametrize("status", ["pending", "restore_raise", "drained_failure"]) +def test_restore_attempt_owns_lease_until_native_receipt_and_never_replays( + native, status +): + state, _, calls = native + owner = storage() + assert owner.store(*vector(owner)).wait(30) + state.mode = status + state.gate = Future() + state.gate.set_result( + SimpleNamespace( + done=status == "drained_failure", success=False, message="injected restore" + ) + ) + operation = owner.load(*vector(owner, (5, 6))) + if status == "drained_failure": + assert not operation.wait(30) + assert owner.pending_bytes == owner.pending_count == 0 + owner.close() + else: + with pytest.raises(CacheRetirementError): + operation.wait(30) + assert owner.pending_bytes == 32 and owner.pending_count == 1 + assert not owner.load(*vector(owner, (5, 6))).wait(0) + with pytest.raises(CacheRetirementError): + owner.close() + assert not any(c[0] == "unregister" for c in calls) + assert [c[0] for c in calls].count("restore") == 1 + assert not any(c[0] == "release" for c in calls) + assert all((t.as_subclass(torch.Tensor) == 91).all() for t in owner.layers.values()) + + +def test_incomplete_restore_lease_released_before_any_gpu_submission(native): + _, data, calls = native + owner = storage() + assert owner.store(*vector(owner)).wait(30) + del data[owner.namespace, "v", bytes([1]) * 32] + assert not owner.load(*vector(owner, (5, 6))).wait(30) + assert [c[0] for c in calls].count("release") == 1 + assert not any(c[0] == "restore" for c in calls) + assert owner.pending_bytes == owner.pending_count == 0 + assert all((t.as_subclass(torch.Tensor) == 91).all() for t in owner.layers.values()) + owner.close()