diff --git a/access.py b/access.py index 2ae09b56..7c8ea13a 100644 --- a/access.py +++ b/access.py @@ -230,10 +230,12 @@ def _mint(self, actor_id, kind, scopes, ttl): _json(sorted(scopes)), int(time.time()) + ttl, 0, 0, int(time.time()))) return token, credential_id - def _local_owner_token(self, *, initial=False): + def _local_owner_token(self, *, initial=False, require_existing=False): """Explicit local administration only; never exposed by a transport route.""" with self.lock, self.db.transaction(): owners = self.db.execute("SELECT id FROM access_actors WHERE kind='owner'").fetchall() + if require_existing and not owners: + raise AccessError('An existing local owner is required; bootstrap explicitly first.', 'unavailable', 409) if initial and owners: raise AccessError('Owner already exists; use local login renewal.', 'conflict', 409) if len(owners) > 1: diff --git a/adapters/rheon/cpu_compare.py b/adapters/rheon/cpu_compare.py new file mode 100644 index 00000000..998630dc --- /dev/null +++ b/adapters/rheon/cpu_compare.py @@ -0,0 +1,206 @@ +"""Optional CPU comparison; captured lab algorithms remain unchanged.""" +import argparse +import math +import os +from pathlib import Path +import sys +import statistics +import tempfile + +if __package__: + from . import generate, native_transport +else: + import generate + import native_transport + +CHECKPOINT_SHA256 = '25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9' +BUILD_RECEIPT_SHA256 = '4eb9f3b83c77261bf49ce4f2d5718b615d4d70aeb68513b9eface8ccf03e6620' +FREEZE_SHA256 = '7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba' +BINARY_SHA256 = '583c939548d68ec7b25cf60bb9434ca1416f28ab6c06b3cad68a1aaf10b344db' +SOURCES = { + 'trajectory_cpu.py': 'f8479f2667fd91fe83bf6c2d66fd1fb9cd9b60d960615b25f528ccac700ba039', + 'trajectory_experiment.py': 'e1fd053f693730d39ded6a7ee50fd6e562bef37b2171bb7ed26acb770605e2f6', + 'trajectory_model.py': '83e93e75bbe05a8dfc8e8dc416b333d6d2e2614613ba45b4f77f99bb94f9a0b4', + 'verify_parallel_results.py': '9d4bd626bbd4352102c11765a47040a37e729a15d5074211f5e60b5bc9ab31ad'} +OUTPUTS = [('comparison-freeze.json', 'comparison_configuration', 256*1024), + ('timing.json', 'comparison_timing', 8*1024*1024), + ('result.json', 'comparison_result', 8*1024*1024), + ('input-model.pt', 'source_checkpoint', 1024*1024), + ('input-freeze.json', 'source_freeze', 1024*1024), + ('comparison-receipt.json', 'comparison_receipt', 1024*1024)] + + +def controls(threads, repetitions): + if type(threads) is not int or not 1 <= threads <= 8 or type(repetitions) is not int or not 1 <= repetitions <= 3: + raise ValueError('CPU comparison requires threads 1..8 and repetitions 1..3.') + + +def inputs(checkpoint, freeze, build, native): + original = generate.decode(freeze) + receipt = generate.decode(build) + if (generate.sha(checkpoint) != CHECKPOINT_SHA256 or generate.sha(freeze) != FREEZE_SHA256 or type(original) is not dict + or original.get('checkpoint_sha256') != CHECKPOINT_SHA256 or original.get('n') != 32 + or original.get('test_seeds') != [900, 901, 902, 903] or original.get('test_frames') != 120): + raise ValueError('Checkpoint/freeze differs from admitted comparison.') + if (generate.sha(build) != BUILD_RECEIPT_SHA256 or generate.sha(native) != BINARY_SHA256 or type(receipt) is not dict + or receipt.get('status') != 'built' or receipt.get('binary', {}).get('sha256') != BINARY_SHA256 + or receipt.get('all_locked_archives_verified') is not True + or receipt.get('lock_sha256_before') != receipt.get('lock_sha256_after')): + raise ValueError('Unqualified parallel build receipt.') + for name, pin in generate.SOURCE_PINS.items(): + source = receipt.get('sources', {}).get(name, {}) + if source.get('commit') != pin or source.get('status_before') != '' or source.get('status_after') != '': + raise ValueError('Parallel build source differs.') + return original + + +def bounded(value, limit, strict=False): + if type(value) not in (int, float) or value < 0 or value > 1e100 or not math.isfinite(value) or (value >= limit if strict else value > limit): + raise ValueError('Scientific result failed its numerical gate.') + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def validate_result(result, threads, repetitions): + controls(threads, repetitions) + config = result.get('config', {}) + expected = {'schema': 'rheon-cpu-only-comparison-v1', 'device': 'cpu', + 'all_model_tensors_on_cpu': True, 'cuda_visible_devices': '', 'n': 32, + 'frames': 120, 'seeds': [900, 901, 902, 903], 'repetitions': repetitions, + 'native_threads_all_methods': threads, 'torch_threads': threads, + 'torch_interop_threads': 1, 'checkpoint_sha256': CHECKPOINT_SHA256, + 'binary_sha256': BINARY_SHA256, + 'test_status': 'previously exposed seeds; backend/hardware comparison, no retraining', + 'source_sha256': {key: value for key, value in SOURCES.items() if key != 'verify_parallel_results.py'}} + require(result.get('status') == 'verified' and all(config.get(k) == v for k, v in expected.items()), + 'Incomplete CPU comparison configuration.') + for key in ('model_setup_s', 'warmup_s'): + bounded(config.get(key), 1e100) + for key in ('torch', 'python_executable', 'timing_scope'): + require(type(config.get(key)) is str and bool(config[key]), 'Missing runtime/timing evidence.') + methods = ('jacobi', 'sgs', 'learned') + identities = [(repeat, seed, method) for repeat in range(repetitions) for seed in range(900, 904) + for method in methods[(repeat+seed-900) % 3:]+methods[:(repeat+seed-900) % 3]] + records = result.get('records', []) + require([(r.get('repetition'), r.get('seed'), r.get('method')) for r in records] == identities, + 'Missing, reordered or duplicate comparison trajectory.') + residuals, divergences = [], [] + for record in records: + reports = record.get('reports', []) + require(record.get('frames') == 120 and len(reports) == 120, 'Missing comparison frames.') + bounded(record.get('wall_ms_including_native_startup_teardown'), 1e100) + require(record['wall_ms_including_native_startup_teardown'] > 0, 'Invalid timing.') + for index, report in enumerate(reports, 1): + bounded(report.get('time'), 1e100) + require(report.get('frame') == index and math.isclose(report['time'], index/60, rel_tol=1e-12, abs_tol=1e-12), + 'Native frame/time order differs.') + bounded(report.get('relative_residual'), 1e-8) + bounded(report.get('actual_divergence_max'), 1e-5) + bounded(report.get('courant'), 1) + residuals.append(report['relative_residual']) + divergences.append(report['actual_divergence_max']) + require(record['method'] != 'learned' or report.get('iterations') == 0, + 'Learned path used iterative correction.') + medians = {method: statistics.median(r['wall_ms_including_native_startup_teardown'] for r in records if r['method'] == method) for method in methods} + require(result.get('median_120_frame_wall_ms') == medians, 'Incorrect timing medians.') + for method in ('jacobi', 'sgs'): + require(result.get('speedup_vs_'+method) == medians[method]/medians['learned'], 'Incorrect timing ratio.') + require(result.get('qualified_timed_frames') == len(records)*120 + and result.get('max_relative_residual') == max(residuals) + and result.get('max_divergence') == max(divergences), 'Incorrect report totals/maxima.') + bounded(result.get('python_peak_rss_bytes'), 1e100) + accuracy = result.get('accuracy', []) + require([case.get('seed') for case in accuracy] == list(range(900, 904)), 'Missing accuracy replay.') + for case in accuracy: + for method in ('jacobi', 'sgs'): + value = case.get(method, {}) + frames = value.get('frames', []) + require(len(frames) == 120, 'Missing accuracy frames.') + for field in ('velocity', 'tracer'): + for frame in frames: + bounded(frame.get(field+'_relative_l2'), 1e-5, True) + maximum = max(frame[field+'_relative_l2'] for frame in frames) + require(value.get('max_'+field+'_relative_l2') == maximum, 'Incorrect accuracy maximum.') + + +def validate_artifacts(config, timing, result, threads, repetitions): + validate_result(result, threads, repetitions) + require(config == timing.get('config') == result['config'], 'Configuration artifacts differ.') + require(set(timing) == set(result) - {'accuracy', 'status'}, 'Timing artifact fields differ.') + for key, value in timing.items(): + if key != 'python_peak_rss_bytes': + require(result[key] == value, 'Timing and final result differ.') + bounded(timing.get('python_peak_rss_bytes'), 1e100) + require(result['python_peak_rss_bytes'] >= timing['python_peak_rss_bytes'], 'Peak RSS decreased.') + + +def run(args): + controls(args.threads, args.repetitions) + root = Path(__file__).resolve().parent + scripts = root/'lab/scripts' + for name, digest in SOURCES.items(): + if generate.sha(generate.capture(scripts/name, 65536)[0]) != digest: + raise ValueError('Captured lab algorithm changed.') + checkpoint = generate.capture(args.checkpoint, 1024*1024)[0] + freeze = generate.capture(args.freeze, 1024*1024)[0] + build = generate.capture(args.build_receipt, 1024*1024)[0] + binary = root/'lab/target/release/rheon-trajectory-parallel' + inputs(checkpoint, freeze, build, generate.capture(binary, 8*1024*1024)[0]) + policy = generate.capture(args.native_policy, 1024*1024)[0] + transport = native_transport.Transport(binary, policy) + output = Path(args.output) + if not output.is_dir() or any(output.iterdir()): + raise ValueError('Expected an empty admitted output directory.') + os.environ['CUDA_VISIBLE_DEVICES'] = '' + sys.path.insert(0, str(scripts)) + import trajectory_cpu + trajectory_cpu.exp.subprocess = transport + with tempfile.TemporaryDirectory(prefix='cpu-comparison-') as temporary: + work = Path(temporary) + original = work/'original'; original.mkdir() + (original/'model.pt').write_bytes(checkpoint) + (original/'freeze.json').write_bytes(freeze) + result_path = work/'comparison' + argv = sys.argv + try: + sys.argv = ['trajectory_cpu.py', str(original), str(result_path), '--threads', str(args.threads), '--repetitions', str(args.repetitions)] + trajectory_cpu.main() + finally: + sys.argv = argv + result_raw = generate.capture(result_path/'result.json', 8*1024*1024)[0] + validate_result(generate.decode(result_raw), args.threads, args.repetitions) + artifacts = {'result.json': result_raw, 'input-model.pt': checkpoint, 'input-freeze.json': freeze, + 'comparison-freeze.json': generate.capture(result_path/'freeze.json', 256*1024)[0], + 'timing.json': generate.capture(result_path/'timing.json', 8*1024*1024)[0]} + validate_artifacts(generate.decode(artifacts['comparison-freeze.json']), + generate.decode(artifacts['timing.json']), generate.decode(result_raw), args.threads, args.repetitions) + receipt = {'schema_version': 1, 'comparison': 'fresh_cpu_checkpoint_comparison', + 'limitations': ['Previously exposed seeds; no retraining or GPU acceptance.', + 'Native integrity launch overhead is included in original wall timings.', + 'Full accuracy fields are compared in memory, not retained by this recipe.'], + 'source_sha256': SOURCES, 'build_receipt': generate.decode(build), + 'native_policy': generate.decode(policy), 'native_launches': transport.evidence, + 'outputs': {name: generate.sha(raw) for name, raw in artifacts.items()}} + artifacts['comparison-receipt.json'] = generate.encode(receipt) + for name, _role, limit in OUTPUTS: + if len(artifacts[name]) > limit: + raise ValueError('Comparison output exceeded admitted budget.') + for name, raw in artifacts.items(): + with (output/name).open('xb') as handle: + handle.write(raw); handle.flush(); os.fsync(handle.fileno()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('checkpoint', 'freeze', 'build-receipt', 'native-policy', 'output'): + parser.add_argument('--'+name, required=True) + parser.add_argument('--threads', type=int, default=8) + parser.add_argument('--repetitions', type=int, default=1) + run(parser.parse_args()) + + +if __name__ == '__main__': + main() diff --git a/adapters/rheon/cpu_project.py b/adapters/rheon/cpu_project.py new file mode 100644 index 00000000..62aa5e93 --- /dev/null +++ b/adapters/rheon/cpu_project.py @@ -0,0 +1,93 @@ +"""Assemble an explicit CPU comparison project without executing model code.""" +import argparse +import os +from pathlib import Path +import shutil +import uuid + +if __package__: + from . import cpu_compare, generate +else: + import cpu_compare + import generate + + +def definition(sources, runtime_snapshot_sha256): + if type(runtime_snapshot_sha256) is not str or not generate.HASH.fullmatch(runtime_snapshot_sha256): + raise ValueError('A qualified runtime snapshot SHA256 is required.') + native = 'lab/target/release/rheon-trajectory-parallel' + arguments = [] + for flag, token in [('checkpoint', {'input': 'checkpoint'}), ('freeze', {'input': 'freeze'}), + ('build-receipt', {'input': 'build_receipt'}), ('native-policy', {'native_policy': native}), + ('threads', {'parameter': 'threads'}), ('repetitions', {'parameter': 'repetitions'}), + ('output', {'output_dir': True})]: + arguments.extend([{'literal': '--'+flag}, token]) + return {'schema_version': 1, 'format': 'external_recipe_v1', 'id': 'rheon-cpu-comparison', + 'name': 'Rheon frozen checkpoint CPU comparison', 'version': 1, + 'runtime': 'captured_cpu_python_v1', 'runtime_snapshot_sha256': runtime_snapshot_sha256, + 'sources': [{'path': name, 'sha256': generate.sha(raw)} for name, raw in sorted(sources.items())], + 'native_sources': [native], 'entrypoint': 'cpu_compare.py', 'arguments': arguments, + 'parameters': {'threads': {'type': 'integer', 'min': 1, 'max': 8, 'default': 8}, + 'repetitions': {'type': 'integer', 'min': 1, 'max': 3, 'default': 1}}, + 'inputs': {name: {'max_bytes': 1024*1024} for name in ('checkpoint', 'freeze', 'build_receipt')}, + 'outputs': [{'path': name, 'role': role, 'max_bytes': limit, 'required': True} + for name, role, limit in cpu_compare.OUTPUTS], + 'environment': {}, 'resources': {'wall_seconds': 600, 'max_output_bytes': 32*1024*1024}} + + +def assemble(lab_root, binary, build_receipt, runtime_snapshot_sha256, destination): + lab = Path(lab_root) + native = generate.capture(binary, 8*1024*1024)[0] + build = generate.capture(build_receipt, 1024*1024)[0] + checkpoint = generate.capture(lab/'experiments/trajectory-v2/model.pt', 1024*1024)[0] + freeze = generate.capture(lab/'experiments/trajectory-v2/freeze.json', 1024*1024)[0] + cpu_compare.inputs(checkpoint, freeze, build, native) + sources = {'lab/target/release/rheon-trajectory-parallel': native} + for name, digest in cpu_compare.SOURCES.items(): + raw = generate.capture(lab/'scripts'/name, 65536)[0] + if generate.sha(raw) != digest: + raise ValueError('Original lab source identity differs.') + sources['lab/scripts/'+name] = raw + for name in ('cpu_compare.py', 'native_transport.py', 'generate.py'): + sources[name] = generate.capture(Path(__file__).with_name(name), 128*1024)[0] + recipe = generate.encode(definition(sources, runtime_snapshot_sha256)) + data = {'checkpoint': ('model.pt', checkpoint), 'freeze': ('freeze.json', freeze), + 'build_receipt': ('build-receipt.json', build)} + manifest = {'schema_version': 1, 'id': uuid.uuid4().hex, 'name': 'Rheon CPU checkpoint comparison', + 'data': [{'id': key, 'label': key, 'path': name, 'sha256': generate.sha(raw)} + for key, (name, raw) in data.items()], + 'recipes': [{'id': 'compare', 'label': 'Compare frozen checkpoint on CPU', + 'path': 'recipe.json', 'sha256': generate.sha(recipe)}], + 'evidence': [], 'requirements': [{'id': 'captured-cpu-runtime', + 'label': 'Qualified captured CPU runtime; previously exposed seeds; no training or GPU acceptance'}]} + files = {**sources, **{name: raw for name, raw in data.values()}, + 'recipe.json': recipe, 'project.json': generate.encode(manifest)} + destination = Path(destination) + destination.mkdir(mode=0o700, exist_ok=False) + try: + for name, raw in files.items(): + path = destination/name + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('xb') as handle: + handle.write(raw); handle.flush(); os.fchmod(handle.fileno(), 0o400); os.fsync(handle.fileno()) + for directory in [*sorted((p for p in destination.rglob('*') if p.is_dir()), reverse=True), destination]: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + except BaseException: + shutil.rmtree(destination) + raise + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('lab-root', 'binary', 'build-receipt', 'runtime-snapshot-sha256', 'destination'): + parser.add_argument('--'+name, required=True) + args = parser.parse_args() + result = assemble(args.lab_root, args.binary, args.build_receipt, args.runtime_snapshot_sha256, args.destination) + print(generate.encode({'project_id': result['id'], 'manifest': str(Path(args.destination)/'project.json')}).decode()) + + +if __name__ == '__main__': + main() diff --git a/adapters/rheon/native_transport.py b/adapters/rheon/native_transport.py new file mode 100644 index 00000000..a32d34e8 --- /dev/null +++ b/adapters/rheon/native_transport.py @@ -0,0 +1,93 @@ +"""Launch-only replacement for the captured lab's subprocess dependency.""" +import io +import os +from pathlib import Path +import struct +import subprocess + +if __package__: + from . import generate +else: + import generate + + +class PrefixReader: + def __init__(self, prefix, stream): + self.prefix = io.BytesIO(prefix) + self.stream = stream + + def read(self, size=-1): + if size < 0: + raise ValueError('Native reads must be bounded.') + head = self.prefix.read(size) + return head + self.stream.read(size - len(head)) if len(head) < size else head + + def close(self): + self.prefix.close() + self.stream.close() + + +def exact(stream, size): + raw = bytearray() + while len(raw) < size: + part = stream.read(size - len(raw)) + if not part: + raise ValueError('Truncated native READY.') + raw.extend(part) + return bytes(raw) + + +class Transport: + PIPE = subprocess.PIPE + + def __init__(self, binary, policy_raw): + self.binary = str(Path(binary)) + self.native, _ = generate.capture(binary, 8 * 1024 * 1024) + self.policy, self.observed = generate.native_policy(policy_raw, self.native) + self.evidence = [] + + def Popen(self, args, **kwargs): + if (type(args) is not list or len(args) != 4 or args[0] != self.binary + or args[1] != '32' or args[2] not in ('jacobi', 'sgs', 'learned') + or args[3] not in ('900', '901', '902', '903') + or kwargs != {'stdin': subprocess.PIPE, 'stdout': subprocess.PIPE}): + raise ValueError('Native launch differs from admitted CPU comparison.') + if os.path.lexists('/etc/ld.so.preload'): + raise ValueError('System preload is unsupported.') + native_fd = generate.seal('rheon-cpu-native', self.native) + loader_fd = None + process = None + try: + loader_fd = generate.seal('rheon-cpu-loader', self.observed[self.policy['loader']['path']][0]) + command = [f'/proc/self/fd/{loader_fd}', *self.policy['loader_arguments'], + f'/proc/self/fd/{native_fd}', *args[1:]] + process = subprocess.Popen(command, pass_fds=(native_fd, loader_fd), **kwargs) + header = exact(process.stdout, 4) + length, = struct.unpack(' 65536: + raise ValueError('Oversized native READY.') + raw = exact(process.stdout, length) + ready = generate.decode(raw) + if (type(ready) is not dict or ready.get('schema') != 'rheon-trajectory-v1' + or ready.get('n') != 32 or ready.get('method') != args[2] + or ready.get('seed') != int(args[3]) or ready.get('dt') != 1/60 + or ready.get('density') != 1.25 or ready.get('spacing') != [1/32]*3 + or ready.get('face_lengths') != [33*32*32]*3): + raise ValueError('Native READY differs from requested controls.') + maps = generate.mapping_evidence(process.pid, self.observed, native_fd, loader_fd) + if len(self.evidence) >= 64: + raise ValueError('Native launch count exceeds admitted comparison.') + self.evidence.append({'method': args[2], 'seed': int(args[3]), 'ready': ready, 'mappings': maps}) + process.stdout = PrefixReader(header + raw, process.stdout) + return process + except BaseException: + if process is not None: + process.kill() + process.wait(timeout=3) + process.stdin.close() + process.stdout.close() + raise + finally: + os.close(native_fd) + if loader_fd is not None: + os.close(loader_fd) diff --git a/docs/contracts/captured-cpu-runtime.md b/docs/contracts/captured-cpu-runtime.md new file mode 100644 index 00000000..de0f9abd --- /dev/null +++ b/docs/contracts/captured-cpu-runtime.md @@ -0,0 +1,104 @@ +# Captured CPU Python runtime + +`captured_cpu_python_v1` is an explicitly captured local CPython 3.12 CPU profile. +It adds no installation, scheduler, network isolation or hostile-code sandbox. +The runtime snapshot and reviewed recipe must remain unchanged for an attempt; +all code shares the local UID. `system_python` recipes keep their existing shape, +limits and behavior. + +## Capture and authority + +`runtime_snapshots.py inspect|create|verify --data CATALOG` is explicit local +administration. Inspection/creation takes a bounded JSON `--config`; creation also +requires `--expected-inventory-sha256 SHA --quiesced`. Source paths and package +entries are explicit. No package hooks, project imports or installations execute +during capture. Required package metadata and licenses remain present. Optional +unsupported components may be excluded only through recorded configuration. + +The CLI requires an existing owner, exchanges a local one-use credential in +memory, and revokes the session on exit. Creation holds the existing Runs runtime +lease and catalog lock; an active application or retained queued/running/cancelling/ +unknown attempt refuses creation. It never reconciles or cancels work implicitly. +Measured copying plus a declared free-space reserve must fit. External disk +writers remain outside this cooperative allocation guarantee. + +The immutable content address is `runtime-snapshots/`. Source +files are streamed into private staging, compared with the reviewed inventory, +normalized to mode0400 (data) or0500 (ELF), synchronized and published without +replacement. An existing snapshot must verify before reuse. Filesystem content +is canonical; there is no mutable snapshot registry. Lost audit acknowledgements +remain visible and do not authorize replacing content. + +## Manifest and descriptor + +Canonical manifest JSON has `schema_version:1`, `profile`, `python_version`, +`packages`, `launch`, sorted `files` and `total_bytes`. Each file has relative +`path`, `bytes`, `sha256`, integer `mode` and `kind` (`elf` or `data`). Launch paths +name `python`, `loader`, `home`, `stdlib`, `dynload`, `packages` and ordered +`library_dirs`. Preserve relative native-library layout and qualify its contained +ORIGIN search paths. Undeclared files, links, special files, changed modes/bytes, +unsupported loader mechanisms and escaping references refuse execution. + +Limits are 40,000 files, 512 MiB per file, 2 GiB total and 16 MiB manifest. These +are maximum bounds, not an allocation reservation. The recipe definition selects +`runtime: "captured_cpu_python_v1"` and `runtime_snapshot_sha256`. Resolution is +only beneath its bound catalog; browser clients cannot choose source directories. + +Material adds `cpu_snapshot`, a bounded descriptor containing manifest identity, +owned root, catalog ID, file/byte counts, Python/package versions and launch paths. +The descriptor and the exact custody module SOURCE become captured inputs at +`runtime-bootstrap/snapshot.json` and `runtime-bootstrap/recipe_process.py`. +Bootstrap SHA256 binds the retained host identity. Cold validation never replaces +it with the currently installed helper source. + +The existing material `runtime` remains the trusted system-Python host closure. +The recipe runtime and `cpu_snapshot` identify the target runtime. The UI displays +the target profile; the complete material remains available for review. Existing +recipes gain no default snapshot fields. + +## Execution and evidence + +The parent verifies the snapshot before launching custody. The helper checks +ordinary captures before READY, then polls cancellation during full snapshot +verification after GO and before forking. The custodied launcher verifies again +before starting captured Python. Snapshot hashing and traversal have a separate +60-second preparation budget; the CPU controller permits62 seconds for readiness. +The old profile retains10/12 seconds. Custodian GO revalidation precedes the child +wall timer; the declared child budget includes launcher checks, Python/package +initialization and recipe execution. Scientific timings use the original script's +own interval. Normal cancellation, child ownership, drainage and unknown-cessation rules +continue to apply. + +Reserved `--cpu-launch` executes the captured loader/interpreter by path, with +fixed captured library search directories, disabled loader cache/hardware-capability +substitution and absent global preload. Path execution is explicitly cooperative, +not a claim that a hostile same-UID writer cannot race verification. + +The captured child uses LANG=C, LC_ALL=C, PYTHONCOERCECLOCALE=0 and PYTHONUTF8=1. +This keeps UTF-8 Python text/filesystem behavior without uncaptured system locale +or gconv cache mappings. The trusted host retains its existing environment. + +`--cpu-child` uses explicit captured PYTHONHOME and `-P -S -B`; validates initial +Python paths and module origins; adds only captured packages; imports NumPy/Torch; +checks versions, CPU-only Torch and actual native mappings; then invokes the +captured recipe. The canonical bounded `tuldok_cpu_runtime` stdout record retains +actual Python/package identity, import locations and mapped libraries. Standard +Run custody retains the stream and its hash. `.pth`, user-site and inherited Python +or loader overrides are excluded. CUDA, JIT/compiler/plugin execution and Torch multiprocessing are not +qualified by this profile. The required torch_shm_manager executable is captured +for normal Torch initialization; its presence alone is not helper-execution +qualification. + +Torch's two synthetic modules, `torch.ops` and `torch.classes`, have pseudo +filenames rather than file origins. Only those exact names, exact defining class +identities and expected pseudo filenames are admitted; their defining modules +must be captured manifest files. Evidence records this provenance separately as +`virtual_modules`. Other relative module origins remain invalid. The bootstrap +record proves the observed initialization state, not continuous monitoring of +every import or mapping during arbitrary recipe execution. + +Complete backup includes snapshot files. Historical completed Runs preserve their +original descriptor and specification after relocation; new execution resolves the +current snapshot root and requires fresh approval. Hash custody is not scientific +acceptance. Actual CPU computation, checkpoint compatibility, numerical gates and +UI workflows require their own qualification evidence. diff --git a/docs/contracts/rheon-cpu-comparison.md b/docs/contracts/rheon-cpu-comparison.md new file mode 100644 index 00000000..216e9e1a --- /dev/null +++ b/docs/contracts/rheon-cpu-comparison.md @@ -0,0 +1,124 @@ +# Optional Rheon checkpoint CPU comparison + +P5a wraps the original lab CPU comparison in the generic external recipe lifecycle. +It uses the existing project references, captured runtime, owner approval and sole +Runs supervisor. It does not register a scheduler, fit a model, change lab +algorithms, or claim to reproduce historical GPU execution. + +## Inputs and assembly + +`adapters/rheon/cpu_project.py` accepts `--lab-root`, `--binary`, +`--build-receipt`, `--runtime-snapshot-sha256`, and `--destination`. +Its Python interface is +`assemble(lab_root, binary, build_receipt, runtime_snapshot_sha256, destination)`. +Assembly validates existing bytes, creates a new directory, refuses overwrite, +and never imports Torch or executes native/model code. + +The project manifest exposes recipe reference `compare`, backed by definition +`rheon-cpu-comparison`, and data references `checkpoint`, `freeze`, and +`build_receipt`. The definition selects `captured_cpu_python_v1` with an explicit +`runtime_snapshot_sha256`; assembly does not qualify a snapshot merely by +accepting a syntactically valid digest. Generic runtime admission verifies it. + +The captured source layout preserves the original scripts: + +- `lab/scripts/trajectory_cpu.py` +- `lab/scripts/trajectory_experiment.py` +- `lab/scripts/trajectory_model.py` +- `lab/scripts/verify_parallel_results.py` +- `lab/target/release/rheon-trajectory-parallel` + +`cpu_compare.py`, `native_transport.py` and the unchanged `generate.py` utility +module accompany them. Original checkpoint and freeze are copied as `model.pt` +and `freeze.json`; the measured build receipt is `build-receipt.json`. Every +source/data reference carries its measured SHA256. Files are created read-only; +source checkouts and original experiment records are unchanged. + +The implementation pins the four original algorithm hashes and these artifacts: + +| Artifact | SHA256 | +| --- | --- | +| Original checkpoint | `25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9` | +| Original freeze | `7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba` | +| Newly built parallel native | `583c939548d68ec7b25cf60bb9434ca1416f28ab6c06b3cad68a1aaf10b344db` | +| Measured parallel build receipt | `4eb9f3b83c77261bf49ce4f2d5718b615d4d70aeb68513b9eface8ccf03e6620` | + +The parallel build uses the admitted clean lab commit +`e8a90bfef6b2955c502008ccc62443e29dd9f2ec`, Rheon commit +`2209012bc5f902693d0f695ec1c2d854126af317`, Rust/Cargo 1.92.0, +and the unchanged lockfile. Locked fetch and subsequent locked offline release +build are explicit prerequisite operations, never recipe side effects. This +binary is a newly measured build, not asserted equal to a historical binary. + +## Execution and native transport + +The wrapper accepts `--checkpoint`, `--freeze`, `--build-receipt`, +`--native-policy`, `--threads`, `--repetitions`, and `--output`. +The output directory must already exist and be empty. Threads are integers 1–8 +(default 8); repetitions are integers 1–3 (default 1). Grid 32³, 120 frames, +seeds 900–903 and the three methods Jacobi, SGS and learned are fixed by the +original freeze and script. The recipe declares 600 seconds and 32 MiB aggregate +output. Timeout is a bounded failure, not permission to shorten trajectories. + +The wrapper hides CUDA before importing the original CPU script, adds only the +captured lab script directory to its import path, and invokes the original +`trajectory_cpu.main` with supported arguments. It creates private temporary +input/output directories because the original script requires new output. +The original model loading, CPU placement, warmup, method rotation, simulation, +inference, accuracy replay and numerical algorithms remain unchanged. + +Only the experiment module's `subprocess` dependency is replaced by a narrow +launch proxy. It accepts the exact captured native path, admitted methods/seeds, +grid and pipe arguments. It seals native and loader bytes, applies the existing +`glibc_x86_64_no_preload_v1` policy, and verifies loaded file mappings at READY +before frame commands. READY must match grid/method/seed, timestep 1/60, density +1.25, spacing and MAC face lengths. Its exact original framed bytes are replayed +to the lab reader. Failed READY or mapping checks kill and reap the immediate +child and close pipes. Descendant custody and wall/cancellation bounds remain +with Runs and its helper; the proxy introduces no second supervisor. + +Integrity launch overhead is included in the original wall timings. Runtime +identity and actual Torch version are recorded. Historical training used +Torch 2.14.0+cu132; a fresh captured CPU comparison on another qualified version +must retain that distinction and never rewrite the historical freeze. + +## Declared outputs and validation + +| File | Role | Maximum | +| --- | --- | --- | +| `comparison-freeze.json` | `comparison_configuration` | 256 KiB | +| `timing.json` | `comparison_timing` | 8 MiB | +| `result.json` | `comparison_result` | 8 MiB | +| `input-model.pt` | `source_checkpoint` | 1 MiB | +| `input-freeze.json` | `source_freeze` | 1 MiB | +| `comparison-receipt.json` | `comparison_receipt` | 1 MiB | + +The first three files preserve the original script's emitted bytes. The next two +preserve exact input bytes. The receipt binds output hashes, all four original +source hashes, measured build evidence, native policy, and each native launch's +READY/mapping evidence. Publication as completed artifacts belongs to Runs. +Partial original timing files cannot bypass final verification. + +The wrapper requires the original `verified` result, complete rotated timed +trajectory coverage, 120 ordered reports per trajectory and frame times. It +checks relative residual ≤1e-8, maximum divergence ≤1e-5, Courant ≤1, and zero +iterative corrections for learned pressure. It recomputes reported maxima, +frame counts, wall-time medians and timing ratios. Configuration must agree +across all three report files; timing/final report fields agree apart from the +expected added accuracy/status and nondecreasing peak RSS. + +All four seeds have separate full-trajectory accuracy replays against Jacobi and +SGS. Each per-frame velocity and tracer relative L2 error must be finite and +strictly below 1e-5; declared maxima must equal the per-frame maxima. Full velocity +and tracer fields are compared in memory by the unchanged original algorithm. +RHS and pressure are not accuracy-scored, and replay arrays are not retained by +this recipe. A cold report audit can check report arithmetic and custody, but +cannot recompute field norms from absent arrays. + +These are previously exposed seeds and a CPU backend/checkpoint comparison. +P5a does not qualify new training, unseen-test generalization, historical timing +reproduction, CUDA/Pumas execution, or durable all-six-field evaluation retention. +Those require subsequent admitted work and actual evidence. Focused synthetic +contract tests and native READY/STOP tests are not model acceptance; actual +completed Runs, independent retained-report audit and UI/restart evidence are +separate qualification gates. diff --git a/docs/plans/generic-scientific-projects/execution-ledger.md b/docs/plans/generic-scientific-projects/execution-ledger.md index 1a7047b0..7b24d3d2 100644 --- a/docs/plans/generic-scientific-projects/execution-ledger.md +++ b/docs/plans/generic-scientific-projects/execution-ledger.md @@ -105,3 +105,32 @@ passes seed100/101, allarrays/slices, review/freeze, cancellation and restart; no-adapter actualstartup also passes. Restore qualification exposed/fixed retained absolute-path validation without rewriting receipt hashes. See P4verification. Commit/push/draft publication follows; exact-head hosted CI gates P5. No merge. + +## 2026-10-13 — P4 hosted qualification and P5a admission + +PR49 at c7c696b passed hosted UI38447405152 and lifecycle38447405078. +Fresh main remains c522bf6; owner PR32/33 and the draft stack remain unmerged. +Created feature/scientific-cpu-comparison-20261013 and admitted the original +checkpoint CPU comparison, exact locked parallel build and captured runtime. + +The parallel native build and actual CPU runtime qualification passed. Root owns +material/Run integration and publication; milestone_audit owns snapshot capture, +mesh_ui the shared helper/bootstrap and runtime probe, recipe_design the optional +adapter, and point_assets the real browser/oracle. mesh_review independently +verified helper behavior and runtime evidence; volume_assets reviewed numerical +adapter gates and planned the subsequent release-consuming fit. Failed bootstrap +attempts remain drained and retained; they are not model acceptance. See +[P5a admission](reports/p5a-admission.md) and +[verification in progress](reports/p5a-verification.md). + + +## 2026-10-13 — P5a actual model/UI qualification + +Original CPU comparison passed through a real approved graph/Run with the unchanged +checkpoint and four lab scripts: 1,440 timed frames, all four accuracy replays, +24 native launches and six hash-verified artifact downloads. Restart preserved +exact bytes; the qualified configuration completed in 542.157760 seconds. Actual +no-adapter generic startup also passed. Independent reviews found no remaining +source or numerical-contract blocker. Full stable-source Python regression is +running; draft publication records that pending gate explicitly. No fresh fitting, +GPU, full retained-field evaluation or historical timing claim. diff --git a/docs/plans/generic-scientific-projects/plan.md b/docs/plans/generic-scientific-projects/plan.md index 581843e2..603dda5f 100644 --- a/docs/plans/generic-scientific-projects/plan.md +++ b/docs/plans/generic-scientific-projects/plan.md @@ -2,9 +2,9 @@ **Plan status:** `Active` -**Current phase:** P3 qualified at published commit 7d59060; P4 bounded native generation admitted on feature/native-scientific-generation-20261013. +**Current phase:** P5a actual captured-runtime and frozen-checkpoint CPU comparison passed; final regression/publication pending on feature/scientific-cpu-comparison-20261013. -**Next slice:** P4 — optional native generation and verified Run artifact import into Data; see [P4 admission](reports/p4-admission.md) for reviewed contracts and exact write boundaries. +**Next slice:** Finish P5a regression/publication, then admit frozen-release training; see [P5a verification](reports/p5a-verification.md). **Acceptance status:** `partial` @@ -116,7 +116,7 @@ Enforce at the service boundary reached by every transport, not only at MCP. Aud | Generic project inspection requires no lab runtime | P1 independence | Project owner | Open external manifest with missing runtime/data; observe no children or source writes | Local app/PostgreSQL/browser | Source-supported; product test pending | | Full lab arrays fit the proposed contract | P2 schema and limits | Data owner | Read representative actual NPZ captures in a separate process; inventory dtype/shape/centering/time and round-trip exact arrays | Actual lab capture bytes + NumPy | Actual two-frame native capture acquired; see P2 admission report | | Native process can be controlled by Runs | P3 execution boundary | Runs owner | Start bounded native probe, cancel during request/output, close pipe, kill supervisor, reconcile descendants | Linux, compatible Rust/Rheon binary | Not executed in planning | -| CPU reproduction is available | P5 acceptance runtime | Lab owner | Inspect installed interpreter/package versions; load hash-verified checkpoint in isolated worker; execute documented CPU control | PyTorch runtime + native build + required inputs | Not qualified; do not auto-install | +| CPU reproduction is available | P5 acceptance runtime | Lab owner | Inspect installed interpreter/package versions; load hash-verified checkpoint in isolated worker; execute documented CPU control | PyTorch runtime + native build + required inputs | P5a frozen-checkpoint CPU comparison qualified locally; fresh fitting/evaluation pending; do not auto-install | | Prior performance numbers transfer | No design depends on this | Qualification owner | Separate matched-hardware benchmark with recorded protocol | Recorded GPU/CPU hardware | Not claimed | ## Evidence And Oracle Plan @@ -212,7 +212,7 @@ Statuses below track the approved staged implementation. Before starting each mi **Acceptance gate:** G3 and P4 UI path. Real generated arrays pass cold generic and producer readers; invalid controls fail before spawn; removal of optional package leaves generic features operational. No fabricated native output or silent re-use of PR32's different fixed profile. -**Status:** `Verifying` +**Status:** `Accepted` ### P5: Surrogate training and autonomous evaluation @@ -224,7 +224,7 @@ Statuses below track the approved staged implementation. Before starting each mi **Acceptance gate:** G4 with real runtime/native solver and actual new outputs, protocol-specific lab verification (CPU correctness separated from CUDA/resource/timing claims as detailed in the evidence report), full-field independent checks, no teacher states in learned rollout, seed/family separation, checkpoint/runtime identity and restart inspection. CPU acceptance is distinct from CUDA acceptance and historical speedup reproduction. Missing runtime or bulk inputs is a named blocker, never substituted with fixtures. -**Status:** `Planned` +**Status:** `Active` ### P6: Agent domain interface diff --git a/docs/plans/generic-scientific-projects/reports/p4-verification.md b/docs/plans/generic-scientific-projects/reports/p4-verification.md index 3b50be7a..5cf38744 100644 --- a/docs/plans/generic-scientific-projects/reports/p4-verification.md +++ b/docs/plans/generic-scientific-projects/reports/p4-verification.md @@ -91,3 +91,7 @@ P5 requires a separately reviewed captured CPU runtime, actual fitting/evaluatio and larger lossless result handling. P6/P7 remain planned. Dataset-workflows A8 positive real-model proposals and original Rheon v2 packet retrieval/qualification remain separate open gates. ASR and general 3D training are explicit extensions. + +## Published-head gate + +Commit `c7c696bda48cc2917253ffbcf4d01f763196eecc` passed hosted UI run 38447405152 and lifecycle run 38447405078. PR49 remains draft and unmerged. P4 is accepted; P5 may proceed on its own branch. diff --git a/docs/plans/generic-scientific-projects/reports/p5a-admission.md b/docs/plans/generic-scientific-projects/reports/p5a-admission.md new file mode 100644 index 00000000..7e8bac04 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a-admission.md @@ -0,0 +1,359 @@ +# P5a admission proposal: frozen-checkpoint CPU comparison + +Admitted on feature/scientific-cpu-comparison-20261013 after PR49 at c7c696b +passed UI 38447405152 and lifecycle 38447405078. Fresh main remains c522bf6. +The explicit locked dependency fetch and offline build below are authorized +prerequisites. No merge/deployment authorization. +This is the first P5 checkpoint, not completion of fresh training or full G4. + +## Scope and labels + +Execute the original pinned lab `scripts/trajectory_cpu.py` comparison through +existing Projects/Pipeline/Access/Runs, using exact historical checkpoint/freeze +bytes and a newly measured CPU runtime. No fitting, parameter selection, +algorithm edits, shortened trajectories or substituted fields. All methods use +the original 32-cubed, 120-frame trajectories for seeds 900–903. These seeds were +previously exposed. Label results a fresh CPU correctness/backend comparison. + +Historical experiment runtime: PyTorch 2.14.0+cu132, CUDA training. Locally +installed candidate: PyTorch 2.8.0+cpu, CPython 3.12. The original CPU comparison +script itself permits this and records actual Torch/device values. The optional +Pumas launcher enforces 2.14 and is NOT this recipe. Never rewrite the historical +freeze, label 2.8 as Pumas acceptance, or claim historical timing reproduction. +Weights-only checkpoint compatibility and actual physics gates remain untested +until this admission executes. + +## Actual inputs and source closure + +Historical source directory: +`/workspace/scratch/tuldok-science-probe/lab/experiments/trajectory-v2/` + +- `model.pt`: 272149 bytes; SHA256 + `25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9`. +- `freeze.json`: 96833 bytes; measure and bind exact original bytes in the project. +- New parallel build receipt, produced by the reviewed build below. It is real + measured build evidence, never copied serial-binary evidence with edited labels. + +The actual CPU script consumes checkpoint and freeze, not a dataset release. +Do not display an unused frozen-dataset dependency. All internally generated +rollouts and their seed/exposure status must be explicit in approval/result scope. + +Capture unchanged Python files under a private source layout: + +| File | Bytes | SHA256 | +| --- | ---: | --- | +| trajectory_cpu.py | 6463 | f8479f2667fd91fe83bf6c2d66fd1fb9cd9b60d960615b25f528ccac700ba039 | +| trajectory_experiment.py | 14799 | e1fd053f693730d39ded6a7ee50fd6e562bef37b2171bb7ed26acb770605e2f6 | +| trajectory_model.py | 2823 | 83e93e75bbe05a8dfc8e8dc416b333d6d2e2614613ba45b4f77f99bb94f9a0b4 | +| verify_parallel_results.py | 3368 | 9d4bd626bbd4352102c11765a47040a37e729a15d5074211f5e60b5bc9ab31ad | + +`verify_parallel_results.py` supplies `metrics()` transitively; the original CPU +script omits it from its own source-hash receipt, so outer Tuldok material must +include it. Also capture all optional adapter helpers, the parallel binary, and +its native policy. The runtime owner captures Python/stdlib/NumPy/Torch/transitive +package and ELF identity. No imports from the live lab checkout. + +Maintain layout `lab/scripts/{four files}` and +`lab/target/release/rheon-trajectory-parallel` within captured source, so original +ROOT/BINARY calculations and source hashes remain valid. Binary stays mode0400. +The CPU path does not invoke `trajectory_experiment.provenance()` or its Git +command; no fabricated Git checkout or monkeypatch of provenance is needed here. + +## Optional adapter architecture + +Proposed new files: +- `adapters/rheon/cpu_compare.py`: strict CLI, unchanged lab module invocation, + result verification and output publication into the provided staging directory. +- `adapters/rheon/native_transport.py`: narrowly scoped Popen-compatible transport + for the pinned Native protocol, reusing existing optional sealing/policy/map + utilities rather than duplicating them. +- `adapters/rheon/cpu_project.py`: new-directory-only explicit project assembler. +- `docs/contracts/rheon-cpu-comparison.md` and focused adapter tests. + +Proposed CLI: +`cpu_compare.py --checkpoint PATH --freeze PATH --build-receipt PATH + --native-policy PATH --threads 8 --repetitions 1 --output DIRECTORY` + +The assembler accepts explicit lab root, built parallel binary, measured build +receipt, original checkpoint/freeze and fresh destination. It only copies verified +bytes and creates generic definitions; no source checkout edits, builds, model +loads or overwrite. Native source/policy declarations reuse P4 generic contracts. +Runtime snapshot/profile integration remains the root/runtime owner's work. + +The wrapper sets CPU-only environment before importing any Torch-dependent lab +module. Import only the captured four-script directory under the qualified +runtime bootstrap. Invoke the original `trajectory_cpu.main()` with its original +argv contract; preserve all source bytes, fitting/model/metrics code and asserts. + +Replace only the `trajectory_experiment` module's local subprocess dependency +with a small proxy implementing the exact Native Popen call. Do not monkeypatch +the global subprocess module or rewrite algorithms. The proxy must: +1. Accept only the expected captured binary path, qualified n/method/seeds and + original pipe arguments; reject additional commands/kwargs. +2. Execute sealed native/loader bytes with the approved loader policy, explicit + CPU/thread environment and existing helper descendant custody. +3. Consume the bounded READY envelope, validate it and observe loaded mappings + before any frame command; replay those exact READY bytes through a prefix + reader so the original Native.read_json() observes the unchanged wire. +4. Expose original stdin, read/close-compatible stdout and Popen poll/wait semantics. + Keep descriptor/process cleanup bounded; unknown descendant drainage remains + solely the Runs helper's responsibility. + +All integrity/setup overhead around native launch falls inside original rollout +wall measurements. Disclose this additional overhead; do not compare new timings +as though the historical process boundary was identical. No second scheduler or +producer-specific generic-core branch. + +## Locked native build prerequisite (not yet executed) + +Pinned lab: e8a90bfef6b2955c502008ccc62443e29dd9f2ec. +Pinned Rheon: 2209012bc5f902693d0f695ec1c2d854126af317. +Existing verified scratch toolchain: Rust/Cargo 1.92.0. +Manifest: `lab/probes/trajectory-parallel/Cargo.toml`. +Target source: `../trajectory/src/main.rs`, unchanged; Rheon feature `parallel`. +Output: scratch `target/release/rheon-trajectory-parallel` (currently absent). +Lock SHA256: 6e75b61a8b16dce290081b47c9d0cdb9d21a96c4d53b0385898bc05001307e98. + +After explicit admission, use task-local PATH/RUSTC/CARGO_HOME/CARGO_TARGET_DIR +pointing to the existing scratch toolchain/cache/target, with no global config or +package install. Exact commands, from `/workspace/scratch/tuldok-science-probe`: + +``` +cargo fetch --locked --manifest-path lab/probes/trajectory-parallel/Cargo.toml +cargo build --release --locked --offline --manifest-path lab/probes/trajectory-parallel/Cargo.toml +``` + +Network fetch is a separately visible dependency step, not a recipe side effect. +Verify source cleanliness and lock hash before/after. Check every registry archive +against the existing lock checksum; do not change versions, relax --locked or +replace unavailable crates. Record command, source/tree hashes, toolchain/archive +identity, new binary hash, native dependencies and build log. No assumption of +binary equality with the historical parallel build. + +Missing cache archives (all expected checksums already in that exact lock): +`crossbeam-deque 0.8.8`, `crossbeam-epoch 0.9.21`, `crossbeam-utils 0.8.23`, +`either 1.19.0`, `proc-macro2 1.0.107`, `quote 1.0.47`, `rayon 1.11.0`, +`rayon-core 1.13.0`, `serde 1.0.229`, `serde_derive 1.0.229`, `syn 3.0.6`, +`unicode-ident 1.0.26`. Existing cached archives checked against the lock matched. + +## Wall budget and output contract + +Keep the current 600-second helper wall limit initially. First qualify the +original script's supported `--threads 8 --repetitions 1`: all four seeds and all +120 frames remain; only benchmark repetitions differ, and the result says so. +Original script supports threads1..32 and repetitions1..10; proposed UI exposes +threads1..8/repetitions1..3, default8/1. No test-based runtime tuning claim. + +Historical CPU medians sum to about26.457 seconds per three-method seed. +One repetition across four seeds plus separate full-field accuracy replays is +about212 seconds before setup/capture/adapter overhead. Three repetitions plus +replay is about423 seconds. These are feasibility estimates from another runtime, +not current-host guarantees. Do not silently raise timeout or shorten trajectories. +If the actual complete one-repetition job exceeds600 seconds, retain bounded +failure evidence and re-admit a measured longer profile or explicitly separated +comparison steps before retry. Attempt three repetitions only after current-host +one-repetition measurements support it. + +Use a fresh private work directory outside the flat published-output directory; +the original script requires its output destination not to exist. Copy or move +completed original files into flat artifacts without changing bytes: + +| Artifact | Role | Bound | +| --- | --- | ---: | +| comparison-freeze.json (original freeze.json output) | comparison_configuration | 256KiB | +| timing.json | comparison_timing | 8MiB | +| result.json | comparison_result | 8MiB | +| input-model.pt | source_checkpoint | 1MiB | +| input-freeze.json | source_freeze | 1MiB | +| comparison-receipt.json | comparison_receipt | 1MiB | + +Aggregate32MiB. Preserve original input checkpoint/freeze in published artifacts, +so eventual redundant staging cleanup cannot erase their evidence. Receipt binds +all captured script/input/runtime/build/policy/output identities, transport/map +observations, CPU-only/thread settings, actual Torch version, repetition count, +previously exposed test scope and timing-overhead disclosure. + +Reject incomplete result, unexpected seeds/frame counts, mismatched checkpoint, +missing comparison methods, or failed original physics/accuracy assertions. +Independent qualification recomputes median/report counts and enforces recorded +residual<=1e-8, divergence<=1e-5, Courant<=1, learned iterations0, and both reference +velocity/tracer relative errors<1e-5. Full-field arrays are actually generated and +compared by the original accuracy replay but not retained by this script. Label +that limit; durable all-six-field evaluation retention belongs to P5c, not a false +P5a completion claim. + +## Runtime/storage prerequisite + +No Python package installation is admitted by this plan. Root owns the explicit +captured-runtime snapshot. Existing candidate Torch is2.8.0+cpu; CPython3.12 differs +from current system profile3.13. The largest Torch DSO is427038688B; logical venv +plus base stdlib inventory is1232844222B before remaining external ELF closure. +New generic profile must address file-count/per-file bounds and qualified import +bootstrap, not merely append mutable site-packages. Measure exact closure first. + +Corrected retention applies: all evaluation chunks remain Run artifacts, all +training records reviewed/frozen, representative evaluation Data import only. +Do not require evaluation-release duplicates. Shared runtime, postconfirmed +publication staging cleanup and the owned-QA two-copy backup/restore sequence can +fit current space; no proven capacity blocker. Recheck/reserve actual bytes at +admission instead of treating a2GiB cap as mandatory allocation. + +## Acceptance and dependency order + +P5a: qualify locked parallel build/native loader; runtime identity; real owner +approved graph -> complete CPU comparison -> exact retained artifacts after +restart; cancellation/runtime/input drift fail closed; independent report audit. +Historical input bytes remain identical. Generic no-adapter startup still works. + +P5b after P5a: real generation -> actual training Data review/freeze -> unchanged +ModalPressure.fit using exact reviewed RHS/pressure arrays -> freshCPU checkpoint, +validation/ablation and freshfreeze. Original train() generates data internally; +a release-consuming wrapper is explicitly new orchestration, never an unused +fake dataset node or silent rewrite of original train algorithm. + +P5c after P5b: matching-new-freeze autonomous evaluation with untouched actual +script algorithms; preserve original NPZ/viewport plus all-six-field/wire chunks, +physics and full-field comparison reports. Native learned wire includes the +extra RHS inference request;16-frame stdout+NPY chunks fit40MiB, full pressure +request bytes are retained separately if requested. Shared trajectory families +exclude chunk boundaries. Representative Data/UI import does not require copying +every evaluation chunk into Data and releases. + +PR49 and the locked build prerequisite have passed. Remaining qualification +gates are the captured runtime and real checkpoint/runtime compatibility. +There is no model correctness, performance, GPU or storage-success claim yet. + +## Reviewed captured CPU runtime contract + +Independent review admits a new `captured_cpu_python_v1` profile, retaining all +old `system_python` serialized shapes, bounds and behavior. Snapshot identity +covers runtime bytes only; generated descriptor/bootstrap source is separately +captured afterward, avoiding a recursive source identity. No Python package +installation is admitted. First confirm the pinned native build prerequisite. + +One `runtime_snapshots.py` owner inventories, copies and verifies a finite explicit +runtime configuration into `catalog/runtime-snapshots/`. Preserve +Python/stdlib/package layouts, required metadata/licenses and recursive native +dependencies. Canonical manifest paths are relative; retain size, SHA256, normalized +mode, actual package versions and launch paths. Bounds: 40,000 files, 512 MiB per +file, 2 GiB total and 16 MiB manifest. Stream copying/hashing; reject special files, +escaping links, source changes, ambiguous dependency resolution and unsupported +loader extensions. Publish without replacement only after fsync and verification. +A verified existing content address can be reused; no mutable snapshot registry. + +CLI inspect/create/verify composes Metadata and Access only. Require an existing +owner through a narrow existing-owner option on the local-login method, mint and +exchange credentials only in memory, bind admin authority and revoke on exit. +Never print credentials. Creation requires explicit `--quiesced`, acquires the +existing `workspace_runs` supervisor lease before checking retained attempts and +refuses queued/running/cancelling/unknown Runs. Hold the lease and CatalogLock +through measured free-space checks, copying and publication without a long SQL +transaction. No second scheduler or reservation database. Filesystem content is +canonical; uncertain audit acknowledgement is reported distinctly and never +turns partial files into a published snapshot. External writers remain outside +the cooperative guarantee. + +Recipes select the new profile plus a snapshot hash; resolve it only beneath the +bound catalog. Approval material/spec retain a small hash-bound descriptor, +not tens of thousands of repeated file records. Cold validation is structural +and needs no live packages. A relocated catalog can inspect historical completed +Runs; fresh execution requires a descriptor bound to its current owned root and +fresh approval. Complete filesystem backup includes snapshots automatically. + +`recipe_process.py` owns pure descriptor/manifest validation and streaming byte +verification; snapshot capture calls that same authority. Its existing trusted +system-Python custodian remains unchanged for old profiles. The new profile +captures its bootstrap SOURCE plus descriptor and uses reserved launch modes: +`--cpu-launch` verifies the snapshot and replaces itself with the captured loader +and interpreter; `--cpu-child` establishes exact captured Python paths, validates +bootstrap origins, imports qualified NumPy/Torch, verifies module/native mappings, +then runs the captured recipe entrypoint. No recipe-controlled mode or descriptor +substitution. Explicit PYTHONHOME and `-P -S -B`; no site/.pth/user-site execution, +no inherited Python/loader overrides. Preserve contained ORIGIN resolution and +use fixed captured loader search paths/cache policy with absent global preload. + +Captured interpreter path execution is permitted under the explicit cooperative +same-UID immutable-attempt assumption. It does not prevent hostile concurrent +mutation and is not a sandbox. Native package loading includes ctypes/dlopen; +DT_NEEDED inspection alone is insufficient. Actual CPU import, computation, +checkpoint save/reload and loaded-mapping checks qualify the supported closure. +CUDA, JIT/compiler/plugin paths remain unsupported. Initialization verification +has a separate bounded profile-only preparation budget of at most 60 seconds +(helper) and 62 seconds (controller), selected for its larger 40,000-file closure. +This is a conservative new-profile bound, not an observed old-timer failure. +Include synchronous prelaunch verification +in measured and bounded preparation. The child wall budget includes CPU launcher +verification, imports and recipe execution; only custodian preparation before the +child starts is separate. Original scientific timings remain the script's own +measured interval. Do not silently change old-profile deadlines. + +## Implementation ownership and checks + +Root owns recipe_contract integration and the explicitly reviewed CPU-only Runs timing, +plan/contracts, final tests and publication. A single delegated owner may edit +recipe_process.py and its tests; another owns runtime_snapshots.py, its tests and +the narrow access.py existing-owner change. Optional adapter files/tests are a +separate owner. No overlapping edits or new generic Rheon branches. + +Additional exact write set: runtime_snapshots.py, recipe_contract.py, +recipe_process.py, access.py, focused corresponding tests, optional adapter files +listed above, docs/contracts/captured-cpu-runtime.md, this plan/report directory, +static/workspace-external.js for the displayed target-runtime name, +and Runs/readiness tests for the CPU-only preparation budget. UI changes +remain limited to existing generic result-role projection if required for the +checkpoint workflow. Freeze each shared manifest/descriptor API before parallel +implementation; unsupported dependency or checkpoint compatibility is a concrete +replan trigger, not permission to substitute algorithms. + +Required negative checks cover source/manifest/runtime mutation, missing files, +path/mode/loader escapes, incomplete captures, disk shortage, active scheduler, +authorization and preserved old-profile identities. Actual acceptance requires +original runtime paths absent from Python imports/native mappings, production +custody/cancel/restart, exact historical checkpoint/freeze bytes, original CPU +assertions and independent report audit. Authored fixtures are contract tests only. + +### Cancellation refinement + +The parent verifies the complete CPU snapshot before launching custody. Before +READY, the helper rechecks ordinary captured/host files; it does not repeat the +large snapshot check while cancellation lacks an acknowledged helper identity. +After GO, full snapshot verification polls cancellation between bounded traversal +and hash steps before any recipe fork. The custodied launcher verifies again +before starting captured Python. This preserves pre-execution byte checks while +keeping cancellation responsive; existing profiles are unchanged. + +### Actual bootstrap locale finding + +The first real CPU attempt failed and drained before package/model execution: +LANG=C.UTF-8 mapped host LC_CTYPE and gconv cache files outside the captured +runtime. A direct captured-interpreter/ctypes diagnostic found zero outside +mappings with LANG=C, LC_ALL=C, PYTHONCOERCECLOCALE=0 and PYTHONUTF8=1. Admit +this fixed child-only locale policy, retain it in bootstrap evidence, and keep +the mapping allowlist strict. No new package installation or snapshot recopy. + +### Required Torch helper finding + +The second actual attempt reached Torch import, then failed and drained because +the broad explicit torch/bin exclusion omitted required torch_shm_manager. +Refine exclusions to individual unsupported test executables, capture the required +manager and its native closure, and publish a new immutable snapshot. Preserve +the failed snapshot/catalog and evidence; use a separate final QA catalog so +complete workflow backup contains its own qualified dependencies. Torch +multiprocessing remains unqualified; the admitted lab path uses no such workers. + +### Actual CPU runtime qualification + +The corrected snapshot `cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7` +contains 21,088 files / 880,817,080 bytes. Real production Runs qualified NumPy 2.5.4 +and Torch 2.8.0+cpu under captured CPython 3.12.14: forward/backward arithmetic, +exact NumPy output, weights-only checkpoint reload, cancellation with confirmed +drainage, and cold artifact reopening passed. All 24 observed startup and +post-computation native mappings belong to captured ELF inventory. The two +synthetic Torch module objects bind exact class/name/pseudo-file identities and +captured defining-module provenance; other origins remain strict. + +[Runtime receipt](p5a/runtime-qualification.json) identifies the two successful +qualification attempts. Earlier failed bootstrap attempts are retained and drained. +This is an authored computation qualification, not the historical model comparison. +The original runtime installation was not physically removed during this test; +the observed import and mapping checks establish the narrower recorded result. diff --git a/docs/plans/generic-scientific-projects/reports/p5a-verification.md b/docs/plans/generic-scientific-projects/reports/p5a-verification.md new file mode 100644 index 00000000..11efc42b --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a-verification.md @@ -0,0 +1,103 @@ +# P5a verification — captured CPU comparison + +Candidate branch: `feature/scientific-cpu-comparison-20261013`, based on qualified +PR49 `c7c696b`. The actual model/UI workflow passed; final full regression and hosted CI +remain pending. No merge or deployment. + +## Runtime and native prerequisites + +The locked parallel build succeeded against the pinned lab/Rheon sources, using +Rust/Cargo 1.92.0. Source trees and Cargo.lock remained unchanged; all 18 cached +registry archives matched their lock checksums after the explicit dependency +fetch. The offline build produced a 1,063,328-byte binary with SHA256 +`583c939548d68ec7b25cf60bb9434ca1416f28ab6c06b3cad68a1aaf10b344db`. +[Build receipt](p5a/parallel-receipt.json) and adjacent logs retain the evidence. + +The corrected CPU snapshot contains 21,088 files / 880,817,080 bytes and has +manifest SHA256 +`cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7`. +It was fully verified before atomic publication. No Python packages were installed. +The original failed snapshot remains preserved outside the final QA catalog. + +The [real runtime probe](p5a/runtime-qualification.json) passed through production +preflight, explicit QA owner approval and Runs custody. It executed forward and +backward arithmetic under captured CPython 3.12.14, NumPy 2.5.4 and Torch 2.8.0+cpu; +compared exact NumPy payloads with an independent arithmetic oracle; saved and +weights-only reloaded a checkpoint; cancelled an initialized CPU process with +confirmed drainage; and reopened exact completed artifacts. All 24 observed +startup and post-computation native mappings belonged to captured ELF inventory. +Independent review verified custody, log and artifact hashes. This authored +computation is runtime qualification, not historical model/scientific acceptance. +The original installation was not physically removed during qualification. + +Failed bootstrap attempts remain retained and drained. They exposed uncaptured +locale mappings, an overbroad exclusion of Torch's required shared-memory helper, +and synthetic module origins. The final profile uses a fixed C locale with Python +UTF-8, captures required helper bytes, and admits only the exact two known Torch +synthetic objects with captured defining-module provenance. Ordinary module and +native mapping checks remain strict. Torch multiprocessing is not qualified. + +## Contract and review checks + +- 22 helper tests pass, independently repeated; cancellation during slow GO + verification drains without starting a child. +- 12 snapshot owner tests pass, including authority/lease refusal, drift, capacity, + partial capture, uncertain audit acknowledgement and prepublication verification. +- 4 CPU material tests pass, including old omitted-profile shapes, Unicode paths, + selector bounds and retained bootstrap/host identity. +- 10 optional adapter tests pass with explicit real native inputs, including + sealed READY/STOP and failure cleanup. No fixture is model acceptance. +- 8 retained recipe-contract and 11 external Runs checks passed before the last + helper bootstrap refinements; the final full regression will supersede them. +- Affected JavaScript syntax/controller tests, Python syntax, plan contract and + whitespace checks pass. Independent source/numerical review found no remaining + blocker. No static type checker or Standards Engine certificate is claimed. + +## Actual model/UI qualification + +The real browser passed four gates in +`build/qa/workspace-cpu-comparison/run-VBdQCB`. Run +`0cc3ef6b5f9e460f873c8eda1ba3d525` completed with confirmed drainage in +542.157760 seconds (recorded started-to-finished wall time). +[Browser receipt](p5a/browser-receipt.json), [independent cold audit](p5a/cold-report.json), +four desktop/narrow screenshots and [candidate source hashes](p5a/source-files.json) +are retained here. Six artifacts were downloaded through HTTP and checked against +the public inventory; restart returned identical artifact bytes and material hash. +Project sources and original checkpoint/freeze bytes remained unchanged. + +The unchanged CPU comparison executed all 12 timed trajectories (three methods, +four previously exposed seeds, 120 frames each), plus four complete accuracy +replays: 24 native launches in total. All methods used eight threads and one +benchmark repetition. Maximum residual was 9.98758848e-9, divergence 1.43051147e-6, +and Courant 0.46830165. Maximum reported velocity/tracer relative error was +9.00478467e-8, below the original strict 1e-5 gate. Median 120-frame wall times +were 35.357 s Jacobi, 25.729 s SGS and 4.699 s learned on this host. These include +sealed launch overhead and are observations, not historical timing reproduction. + +The original accuracy replay computed full velocity/tracer norms in memory; +those field arrays are not retained by this comparison. The independent cold +oracle recomputes report coverage/maxima/medians and cross-artifact identities, +not norms from absent arrays. RHS/pressure accuracy is outside this P5a script. +Full-field retention and fresh training remain P5b/P5c. + +Owner-local QA separately read custody receipts and streams and verified their +hash bindings; these are not HTTP log downloads or MCP parity evidence. The +ordinary libc loopback-only canary passed all 12 checks for the UI transport; +this is not a recipe network sandbox. Screenshots were visually inspected at +1440 and 390 pixels without horizontal overflow. Only the observed eight-thread, +one-repetition configuration is qualified here; other allowed settings retain +their declared timeout and are not covered by this observation. + +Generic startup with adapters physically absent passed 10 actual checks in an +isolated source copy: owner authentication, workspaces/workbench, generic project +identity, authored scientific draft import/slice, empty Runs and unchanged copied +sources. [Receipt](p5a/no-adapter-receipt.json) and +[procedure](p5a/no-adapter-procedure.txt) retain the evidence; no snapshot or model +was copied for this check. + +## Publication and remaining scope + +Pending commit, push, draft PR and exact-head CI. P5a does not complete G4 or G8. +Fresh release-consuming fitting, autonomous evaluation, agent interface parity and +final compatibility/UI/restore qualification remain. The independent A8 real-model +proposal and original Rheon v2 packet gates remain open. diff --git a/docs/plans/generic-scientific-projects/reports/p5a/browser-receipt.json b/docs/plans/generic-scientific-projects/reports/p5a/browser-receipt.json new file mode 100644 index 00000000..77c1e33d --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/browser-receipt.json @@ -0,0 +1,121 @@ +{ + "source_commit": "c7c696bda48cc2917253ffbcf4d01f763196eecc", + "dirty": true, + "qualification": "Fresh frozen-checkpoint CPU correctness/backend comparison through actual approved Run", + "previously_exposed_seeds": [ + 900, + 901, + 902, + 903 + ], + "fresh_training_acceptance": false, + "gpu_acceptance": false, + "full_accuracy_fields_retained": false, + "checks": [ + "Actual approved CPU graph completes all original methods, frames and exposed seeds", + "Owner-local QA custody logs retain exact hash-bound CPU bootstrap module and mapping evidence", + "Independent cold audit verifies original checkpoint/source bytes, complete reports, medians and strict accuracy thresholds", + "Server restart preserves exact completed comparison artifacts and caller-owned project inputs" + ], + "captures": [ + { + "file": "cpu-pipeline-1440.jpg", + "width": 1440, + "height": 900, + "qualification": "Actual frozen-checkpoint CPU comparison; exposed seeds, no retraining/GPU acceptance" + }, + { + "file": "cpu-pipeline-390.jpg", + "width": 390, + "height": 900, + "qualification": "Actual frozen-checkpoint CPU comparison; exposed seeds, no retraining/GPU acceptance" + }, + { + "file": "cpu-runs-390.jpg", + "width": 390, + "height": 900, + "qualification": "Actual frozen-checkpoint CPU comparison; exposed seeds, no retraining/GPU acceptance" + }, + { + "file": "cpu-runs-1440.jpg", + "width": 1440, + "height": 900, + "qualification": "Actual frozen-checkpoint CPU comparison; exposed seeds, no retraining/GPU acceptance" + } + ], + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525", + "material_sha256": "abb1b223cbcb86b55a39f6e02d42b6c2cff64d6409d66cb41f2c596ffb254a52", + "runtime_snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7", + "local_qa_evidence": { + "directory": "owner-local-custody", + "custody_receipt_sha256": "89d1062ee70ea2a47eb1006b271280d0e8e48fb567b6b0c52be4dd68ff649318", + "http_download_acceptance": false + }, + "cold": { + "qualification": "Independent retained-report arithmetic and exact input/output custody audit of actual CPU comparison", + "limitations": [ + "Previously exposed seeds 900–903; no retraining or GPU acceptance.", + "Full velocity/tracer norms were computed by the original producer in memory; this cold audit recomputes report maxima, not absent field arrays.", + "Native integrity launch overhead is included; no historical timing reproduction claim." + ], + "artifact_sha256": { + "comparison-freeze.json": "5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076", + "timing.json": "0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b", + "result.json": "d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448", + "input-model.pt": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "input-freeze.json": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "comparison-receipt.json": "d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a" + }, + "torch": "2.8.0+cpu", + "threads": 8, + "repetitions": 1, + "timed_trajectories": 12, + "timed_frames": 1440, + "accuracy_replays": 4, + "native_launches": 24, + "median_120_frame_wall_ms": { + "jacobi": 35357.278365998354, + "sgs": 25729.378062998876, + "learned": 4699.447435497859 + }, + "max_relative_residual": 9.987588478738668e-9, + "max_divergence": 0.000001430511474609375, + "max_courant": 0.4683016538619995, + "accuracy_maxima": { + "900:jacobi:velocity": 8.287529608746873e-8, + "900:jacobi:tracer": 4.881945563597801e-8, + "900:sgs:velocity": 8.554606404046589e-8, + "900:sgs:tracer": 5.004190007866501e-8, + "901:jacobi:velocity": 8.885706621137449e-8, + "901:jacobi:tracer": 5.886900925254406e-8, + "901:sgs:velocity": 9.00478467319294e-8, + "901:sgs:tracer": 5.908868728450394e-8, + "902:jacobi:velocity": 8.43263175826526e-8, + "902:jacobi:tracer": 5.219647297987591e-8, + "902:sgs:velocity": 8.148838940891793e-8, + "902:sgs:tracer": 4.292791785600593e-8, + "903:jacobi:velocity": 8.229298707876792e-8, + "903:jacobi:tracer": 4.629560023953615e-8, + "903:sgs:velocity": 8.495464706555917e-8, + "903:sgs:tracer": 4.8583669586920674e-8 + } + }, + "offline_canary": { + "qualification": "ordinary libc loopback-only transport", + "checks": [ + "connect denied 192.0.2.1", + "sendto denied 192.0.2.1", + "connect denied 2001:db8::1", + "sendto denied 2001:db8::1", + "resolution denied 192.0.2.1", + "resolution denied 2001:db8::1", + "resolution denied external-test.invalid", + "tcp local 127.0.0.1", + "udp local 127.0.0.1", + "tcp local ::1", + "udp local ::1", + "unix local" + ] + }, + "browser": "Chrome/151.0.7922.173" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/cold-report.json b/docs/plans/generic-scientific-projects/reports/p5a/cold-report.json new file mode 100644 index 00000000..6cd3f971 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/cold-report.json @@ -0,0 +1,49 @@ +{ + "qualification": "Independent retained-report arithmetic and exact input/output custody audit of actual CPU comparison", + "limitations": [ + "Previously exposed seeds 900\u2013903; no retraining or GPU acceptance.", + "Full velocity/tracer norms were computed by the original producer in memory; this cold audit recomputes report maxima, not absent field arrays.", + "Native integrity launch overhead is included; no historical timing reproduction claim." + ], + "artifact_sha256": { + "comparison-freeze.json": "5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076", + "timing.json": "0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b", + "result.json": "d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448", + "input-model.pt": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "input-freeze.json": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "comparison-receipt.json": "d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a" + }, + "torch": "2.8.0+cpu", + "threads": 8, + "repetitions": 1, + "timed_trajectories": 12, + "timed_frames": 1440, + "accuracy_replays": 4, + "native_launches": 24, + "median_120_frame_wall_ms": { + "jacobi": 35357.278365998354, + "sgs": 25729.378062998876, + "learned": 4699.447435497859 + }, + "max_relative_residual": 9.987588478738668e-09, + "max_divergence": 1.430511474609375e-06, + "max_courant": 0.4683016538619995, + "accuracy_maxima": { + "900:jacobi:velocity": 8.287529608746873e-08, + "900:jacobi:tracer": 4.881945563597801e-08, + "900:sgs:velocity": 8.554606404046589e-08, + "900:sgs:tracer": 5.004190007866501e-08, + "901:jacobi:velocity": 8.885706621137449e-08, + "901:jacobi:tracer": 5.886900925254406e-08, + "901:sgs:velocity": 9.00478467319294e-08, + "901:sgs:tracer": 5.908868728450394e-08, + "902:jacobi:velocity": 8.43263175826526e-08, + "902:jacobi:tracer": 5.219647297987591e-08, + "902:sgs:velocity": 8.148838940891793e-08, + "902:sgs:tracer": 4.292791785600593e-08, + "903:jacobi:velocity": 8.229298707876792e-08, + "903:jacobi:tracer": 4.629560023953615e-08, + "903:sgs:velocity": 8.495464706555917e-08, + "903:sgs:tracer": 4.8583669586920674e-08 + } +} diff --git a/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-1440.jpg b/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-1440.jpg new file mode 100644 index 00000000..a86b84df Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-390.jpg b/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-390.jpg new file mode 100644 index 00000000..56e8baf1 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p5a/cpu-pipeline-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-1440.jpg b/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-1440.jpg new file mode 100644 index 00000000..20560b5a Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-390.jpg b/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-390.jpg new file mode 100644 index 00000000..60b2d076 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p5a/cpu-runs-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-procedure.txt b/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-procedure.txt new file mode 100644 index 00000000..7a6de640 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-procedure.txt @@ -0,0 +1,59 @@ +"""Disposable real HTTP qualification; no optional adapter or producer execution.""" +import hashlib,json,os,pathlib,shutil,subprocess,tempfile,time,urllib.request,http.cookiejar +ROOT=pathlib.Path('/workspace/Tuldok-v2'); PY='/workspace/Tuldok/.venv/bin/python' +DEST=pathlib.Path(tempfile.mkdtemp(prefix='tuldok-p5a-no-adapter-')); CODE=DEST/'application'; CODE.mkdir() +def sha(raw): return hashlib.sha256(raw).hexdigest() +inventory={} +for source in sorted(ROOT.glob('*.py')): + raw=source.read_bytes();(CODE/source.name).write_bytes(raw);inventory[source.name]=sha(raw) +shutil.copytree(ROOT/'static',CODE/'static') +for source in sorted((CODE/'static').rglob('*')): + if source.is_file(): inventory[str(source.relative_to(CODE))]=sha(source.read_bytes()) +assert not (CODE/'adapters').exists() +project=DEST/'project';project.mkdir() +env=dict(os.environ);env.pop('PYTHONPATH',None);env.pop('PYTHONHOME',None) +subprocess.run([PY,str(ROOT/'tests/qualify_scientific_fields.py'),'--output',str(project/'fields.zip'),'--build-only'],env=env,check=True,capture_output=True) +raw=(project/'fields.zip').read_bytes();manifest={'schema_version':1,'id':'f'*32,'name':'Generic startup without optional adapter','data':[{'id':'fields','label':'Authored generic fields','path':'fields.zip','sha256':sha(raw)}],'recipes':[],'evidence':[],'requirements':[]} +(project/'project.json').write_text(json.dumps(manifest)) +launcher="import importlib.util,pathlib,sys; assert importlib.util.find_spec('adapters') is None; import app; assert pathlib.Path(app.__file__).resolve().parent == pathlib.Path.cwd(); assert not any(n == 'adapters' or n.startswith('adapters.') for n in sys.modules); print('OPTIONAL_ADAPTER_ABSENT_CONFIRMED',flush=True); app.main()" +log=(DEST/'server.log').open('w+');server=subprocess.Popen([PY,'-u','-c',launcher,'--port','0','--data',str(DEST/'catalog'),'--project-root','generic='+str(project)],cwd=CODE,env=env,stdout=log,stderr=subprocess.STDOUT) +receipt={'qualification':'Actual application source snapshot with adapters physically absent; generic authored field import/inspection. No producer execution, mocks or scientific qualification.','snapshot_root':str(CODE),'source_commit':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'dirty':bool(subprocess.check_output(['git','status','--porcelain'],cwd=ROOT,text=True).strip()),'snapshot_file_hashes':inventory,'adapter_exists':False,'checks':[]} +try: + import re + for _ in range(200): + log.flush();output=(DEST/'server.log').read_text();match=re.search(r'Tuldok: (http://127.0.0.1:\d+)',output) + if match: break + if server.poll() is not None: raise RuntimeError(output) + time.sleep(.1) + else: raise RuntimeError('Startup timed out') + base=match.group(1);assert 'OPTIONAL_ADAPTER_ABSENT_CONFIRMED' in output + jar=http.cookiejar.CookieJar();opener=urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar)) + def request(route,body=None): + headers={'Origin':base};data=None + if body is not None: headers['Content-Type']='application/json';data=json.dumps(body).encode() + with opener.open(urllib.request.Request(base+route,data=data,headers=headers),timeout=30) as response: + raw=response.read();ctype=response.headers.get('Content-Type','');return response.status,json.loads(raw) if 'json' in ctype else raw + login=DEST/'owner-code';subprocess.run([PY,'access.py','login','--data',str(DEST/'catalog'),'--out',str(login)],cwd=CODE,env=env,check=True,capture_output=True) + token=login.read_text().strip();login.unlink();status,identity=request('/api/access/session',{'token':token});token=None;assert status==200 and identity['kind']=='owner' + assert any('HttpOnly' in cookie._rest for cookie in jar) + receipt['checks'].append('Normal owner CLI issuance and real HttpOnly session exchange succeeded without adapters.') + for route in ['/workspaces','/workbench','/workspace-external.js','/scientific-fields.js']: + status,payload=request(route);assert status==200 and payload;receipt['checks'].append(route+' served from copied static sources.') + status,opened=request('/api/workspace/projects',{'root':'generic','manifest_path':'project.json'});assert status==201 and opened['id']==manifest['id'] + status,detail=request('/api/workspace/projects/'+manifest['id']);assert detail['data'][0]['observed_sha256']==sha(raw) + status,record=request('/api/workspace/projects/'+manifest['id']+'/import-data/fields',{'expected_sha256':sha(raw)});assert status==201 and record['kind']=='field_sequence' and record['review']=='draft' + status,rows=request('/api/workbench/records');assert len(rows['items'])==1 + field=record['field_sequence']['manifest']['fields'][0]['id'] + status,inspection=request('/api/workbench/records/'+record['id']+'/scientific?frame=0&field='+field+'&axis=z&index=0');assert status==200 and inspection['field']==field and inspection['values'] and len(inspection['physical_origin'])==3 + assert request('/api/workspace/runs')[1]==[] + receipt['checks'].extend(['Generic project hash inspection succeeded.','Authored generic scientific bundle imported as draft through production admission.','Data record list and exact scientific slice/physical coordinates inspected through production HTTP.','Runs list remains empty; no producer or recipe executed.']) + receipt.update(record_id=record['id'],bundle_sha256=sha(raw),slice_sha256=sha(json.dumps(inspection,sort_keys=True).encode()),field=field) + assert not (CODE/'adapters').exists() + for name,digest in inventory.items(): assert sha((CODE/name).read_bytes())==digest + receipt['checks'].append('Copied source/static bytes unchanged; optional adapters directory remains absent.') + (DEST/'receipt.json').write_text(json.dumps(receipt,indent=2));print('PASS no-adapter generic startup:',DEST);print(json.dumps(receipt['checks'],indent=2)) +finally: + server.send_signal(__import__('signal').SIGINT) + try: server.wait(timeout=15) + except subprocess.TimeoutExpired: server.terminate();server.wait(timeout=5) + log.close() diff --git a/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-receipt.json b/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-receipt.json new file mode 100644 index 00000000..37aaa7a3 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/no-adapter-receipt.json @@ -0,0 +1,132 @@ +{ + "qualification": "Actual application source snapshot with adapters physically absent; generic authored field import/inspection. No producer execution, mocks or scientific qualification.", + "snapshot_root": "/tmp/tuldok-p5a-no-adapter-dt6mczya/application", + "source_commit": "c7c696bda48cc2917253ffbcf4d01f763196eecc", + "dirty": true, + "snapshot_file_hashes": { + "access.py": "50681dd5b2aaba01a311a7465b15161ca264fa4ca9812b95f15914adb5d808f4", + "ai.py": "a3b04a3015496ff3c66c3eb450ccf887bd5726229669ac0515d95a2761ad3a9a", + "ai_codex.py": "debd26342e73a87635d772e3ee59d30bb822877008d667b47997f52e22f94f16", + "ai_http.py": "73ca648fe00e8f5a4022b14b72c80dd695228706a0f2b79aee605a524b8f2585", + "ai_openrouter.py": "70930124fac7ef08941f571c1d4492f72b29aa94807cfd5d66bc32555432b160", + "app.py": "6a336d83e8042c016abb2c21e33771e6f7f2f4107e58880033b1090f41fce353", + "audio_assets.py": "d69bfa94037ea132be4adebba26c4389d58925a408f2fc27c6984a71fc8dc1cd", + "audio_classification_export.py": "0f0643470cfa3b015c2d56c0e8e1dc2ad4944bc825127b9131fbbdd36349052d", + "audio_export.py": "bc20d32089b3a004504c6cb072e18db9e285c9195db84679670fe460ef51cfc9", + "audio_training.py": "fd5a21f1bfbfbbd72f8ba5a3bcc88017e2b8c2f075a7a6506310a01156f771a7", + "backup_metadata.py": "15fced02dbc6da6b035b2ab7fbf5bd368ef862d0d03eb01561781d09fb921f0a", + "bulk_import.py": "d09fe5973d4cd405b99423989a29de91fee710c56523f48f77a1c4e6fb03ed39", + "canonical_release.py": "1f92e2d822602211475a5e7c13198388cc0a36aa36e0872f66d9fd353dcc2061", + "caption_import.py": "e6e16f3a4977f03491d119bd8ef67430526e4fb100a2a4161d600df2d6eb8acc", + "caption_proposals.py": "a74ca7993b6f3b7dc7fb867a871148f04c743c048a1e988ab517554a35ae901c", + "curation.py": "b25576749afd01f9e699133c5db7e6113f9a5f0ad8961dafc4084292896b0049", + "dataset_recipes.py": "78ae918228f15f9d5924985075cb96535b8953b4ad5faa821d0cacd0a50e3c88", + "dataset_releases.py": "97b2045fcfda2adfaedd98cb8d42ccd72f093bab4d12bb902b940ae8a060f5e0", + "frozen_classification.py": "a6a9fc244bcb7de7da5217fb36d5e73bb03ca1756c9c63770c23a78336606649", + "gateway_discovery.py": "8a02e2ef9038859c1accce9ea3e31ea489423f19799a279f4be6f9afd61297d0", + "grounded_candidates.py": "55bf111d7dfe951fea960f347ed4bfb498d15dcc604bf46fb1ae2505cc1354cc", + "image_classification_export.py": "995edefa24fc039505337fd8d2915e45d173c18a03af513d9f5d8eb234225b96", + "image_generation.py": "c3a62b3cf0bf1d6f6d453a63f7f2a7d08de63864d880959bd0eab237d9856858", + "image_training.py": "5a7807f8834f96bf17e1c020357fdb96a7eb1ac94b54a440db80559d9923f99c", + "immutable_assets.py": "672c08d949f964fcd8cebd9a888092ab0163f022542df337e02bcb17f3c477c8", + "mesh_release.py": "09d29a60bc37826b906097debaf7eb8577a425d3d791ff5df8d3b03607b10de1", + "meshes.py": "abeca65f8a39af57352828169f90f767b6bc770fb5407fb3f18af53be8ff84e2", + "metadata.py": "42559a7efd599f637e6e289802fa7400d59b2915edcf64e3260952d59cd7b439", + "migrate_sqlite.py": "be1a79a1b6f5e2c8e4c99862043a5afcd4f30d6539388c1600908f4a4f3f92da", + "native_detection_import.py": "80466232bd51f58bca9a00ef68e278da087b012e7f1b3b1171e6e51db3b8c572", + "native_text_import.py": "8e1d53558dca265812792a9ec0802994cf200fd448b6dd11560667d07f64af2c", + "pipeline.py": "030b08b215b37701ec2b9a4f00240edef05213191b9568abdb2f2f013ae84659", + "pointclouds.py": "02d9eef31661dd5a9f0855f8e59d7b839b0ad1590f53a657123a5d9da789dd9a", + "preferences.py": "d6279b18375fa8695584a02cfe962b56e77cf4f3521c5e33cc4baf09f8e4dab7", + "projects.py": "d147c6bae61068f5517fd15e3512311459cd4de3d6edfdd9ea984783470d432d", + "pumas_gateway_descriptor.py": "b9c4259bda856fd19d1d82b3a2c000a74ab3979832248fd3f1f1d3d6309c0752", + "pumas_operations.py": "e8b8acee54a027633ae39f00a23b3d4845ae474d031690134cafb1c75bf73511", + "pumas_vision.py": "5659c7731b5011ff39a0a9b7c27712382983f16286540f47f6b1337941bbda7d", + "recipe_contract.py": "c59180b3468e9a4891a7dfae9676f1d77a6a9af461260dd9020d042251292c50", + "recipe_process.py": "72f3e292305cedeff136eb2a2592668113eaeb2c1c658d9933ca85865eb8fa39", + "release_validation.py": "a04e0f3c941256422b0c3a8c28665b6fba025df9d3cfa6cd5d5686e2bb8de95b", + "rheon_controls_contract.py": "25ec0f98f9b0ec576f30793cd24c1dd7b4e77fa7a6bfc6ed323271f829f982f4", + "rheon_controls_sequence_contract.py": "5c41f3a3c848940191d05d9b5b01a2a398b0d2a67f23c603ab8e918897bd3b55", + "rheon_sequence_contract.py": "063d17a3bbc92a27c26c0f5dd4588a9478091ebbf53ce0ccb818cee6a9393b98", + "rheon_sequences.py": "9e3013fb1b5f1baf3de73678a26f964bc2f802435e6fff26476f816847fd22a5", + "runs.py": "a6c1d9f91503179e1e722d17ed7c9c3b0b9e9540e4afb6ee979d177c10235562", + "runtime_snapshots.py": "dad9e4ddf9c11c0e1f2d5b9a7afd6877e7be375fe02030c65b684bca2b0ef45e", + "saved_searches.py": "fcd4021fc20702568f1e44a165473aa752b9a9794a958ff31b33c42aba7114d5", + "saved_selections.py": "4e89ee272fc8a685ec76b3737879d6560661a73e03fca264c2057785905418a3", + "scientific_fields.py": "9efd0400dbb21285da2b8802397b56cd2f0baaf85e24ed044f756a0e568824ac", + "scientific_release.py": "b419730e7cc00212632b3adcdf63997343b6fb9aecd07a6ad2cb6f5c1728fb60", + "sequence_assets.py": "3ac8361f5e24f6b2af93dbd9030bab864da7bddb32da975d937e852d89fff98d", + "sequence_batch_import.py": "3e7af45744ae7f8393059715dbf7012567d0bbb528c3682dd1b992141ae32b10", + "sequence_release.py": "3d1bac3c2d78d75e08337b0659e435fc81c32c7625d8244be4d41f2ecb9e831c", + "spatial_assets.py": "fdd3715025426024a4c8fa1704645d98940cab7e6002846b9e72a54352fe6ef8", + "spatial_release.py": "4d6ec06fceb72257a9d25da09ab24bcf13fc38f66ae50e8b39d5a5796f9b357c", + "synthetic.py": "dc55737b0f55ff4ec8df56a3d6b4f4596f5067bfd75672ba6326453047cab5e9", + "text_classification_export.py": "2510ad278a95892d4453ffb5b146ae47c11f8e31f3dc4cf9fb005eec79270bdf", + "text_classification_proposals.py": "6a29dfe752be4ae2c69d7b2e4ca3e68e87dc511cfee2429093748b4ccc56c86d", + "text_corpus_export.py": "3af4883df1b714519fb4e9be872f39de20dc3013deca720e32a89789cbdbff69", + "text_training.py": "9794c343af00854519be0873bad766eadf681c4826be010efb3423cae1d0974e", + "upgrade_metadata.py": "df798ada0b929fd329c42add8545109422b5821874aeb4c913c4ef5290015525", + "volume_assets.py": "1e057fcec996969e0a9b44ae7561fb7b6ea95b6201a23617fdbbc3a046b610c1", + "workbench.py": "71bbbaf56ae138c5a80ef4ee9854851470bae5effd20315c8fc4febc795fa643", + "static/app.js": "b82969e67e7a7792a11f239eff2fec959158e1dd78d08a84d68daedb93aad6bb", + "static/audio-editor.css": "6c6cecd83f78d0a37af9011f5a3716dec6d41686e7de85af7e8b3d9e171bd628", + "static/audio-editor.js": "d397bb98480077326b4c96035d891f1d8eb77cef8a04f3b2aef1eb34d5bbcf40", + "static/auth.css": "4ad4c7e8e4aaa04b4837698493b59365859b89f2cedd8717492ba6b57615f620", + "static/auth.js": "6a38fa7dabf57b2c99d293c6851a732af76a19193bea43f7d2a8b8460d71faec", + "static/bulk_import.js": "2d19443342dad5d083ad66e32f5b4877a825296f1a41d8996befd336cec7d367", + "static/caption-proposals.js": "09906d8ea799ee7def657a5d1c7746a5dcbe25797f7849c7ac27364089a083d7", + "static/caption_import.js": "280a116a086c3e8c69fcd17e24aba6630bb6c3d3c1e653a5ed84c184895ae850", + "static/curation.js": "2f51c2e063eddbe16cb92dfcaa96c35c749dd61960ebd92c8c538bf0c6ff389a", + "static/index.html": "34ef87f168f4f1cd291be6c86464e02969b3c760316f0a59f01f50890ec973b3", + "static/instruction-responses.js": "3fb4fb8d78154bedb37ef90afbcc269fdab0938c4ac50236f38a7f26f54ac940", + "static/meshes.css": "0658afff9e77542442b1c5bf1c10ce50603fab4beec1f812ac40a6574f81076e", + "static/meshes.js": "c2eb7622dd4e0a172db9db36e46a0ce66800d5b03aca989cd1f08dd40b8558c8", + "static/native_detection_import.js": "b07d3a9663d5b1534fca24dc7481caf4cb166bb4a83c4c97c2a78fb9c8d50d82", + "static/native_text_import.js": "1704549e9e5cf489e6351d5126858eb774ddd94d060e596a0bfba5fc00ea8773", + "static/pipeline-editor.js": "2272f1a9b106945203112a46d73643cc5d4dbbfd92f0d7c09d1cc36a713042d2", + "static/pointclouds.js": "34851447f34ea0204887b6aae4a86d73f6f94c8d1726b8e82b892027336c0c84", + "static/preferences.js": "fb22c04b6856d33b7e298aa1989d073c937ab40020025f39b86daaff62b948f7", + "static/pumas-gateways.js": "02df8f34362fc2e32f6a4582a9cb71adaa5d7b83ccd897803b9e1d02c7bcb932", + "static/pumas-typed.js": "50fd9fc961dafdd0c34cfcb1137d04f85a99f324f76fd5217f227b789c8835ad", + "static/rights-note.js": "fd09cef5c52b2635ad42998483c7054260140459444fbd948bb596e5966feb11", + "static/runs-view.js": "7568b843ad9b56c1c6878a9756553cdf8092dea3b954aec1947b8f32d730ca85", + "static/saved-searches.js": "49210b9da9dd7f7372e977c15543a77c5ada1df82ad9abdcb350026e16517892", + "static/saved-selections.js": "2c2feecca798217644247f98aab8e7a00d707a25d4af75e02f9d906ccbd905a3", + "static/scientific-fields.js": "96e6289ade2a634e9648cbe8aa899e5bfaddacddcea120835bda8cb6d65e14a6", + "static/sequence-batch.js": "36bbbdc07cc3a03856e6072e4141545c1c0f202805faedec76a2c7f20c0850f7", + "static/sequences.js": "0f0ce113a0e2e3e3160f063ed577f882fb7d53d156f517ed46920030f1c0d17e", + "static/style.css": "25103299f9add27eafbee40c9a12634d7640a6c54a785c981066b6403de0603d", + "static/text-classification-proposals.js": "03dbec1114700f383f8bc9883c8a4cdaae59d898fc44240022b2011893965d6f", + "static/volume-editor.js": "d48806cd446ea05586b524c24340d87abe8c69063ea6b15c254f76b3e51c5db8", + "static/workbench.css": "669aab1c0c8d680c60848ae96ec1925151ea663238d4412d45862ef60112391e", + "static/workbench.html": "6d1ab570aae02f781b905ffa2b6970843c297d6ffe671e8fe7cca620e2a46270", + "static/workbench.js": "7e31b8a0e916f1627cb523d84c6d98c732eb739617b3f3856cbc1c235a6829b2", + "static/workspace-external.js": "a28436e15312f5f67203dc6c2d2c618421b46ace42bd32cad4000df3794c06d2", + "static/workspace-fixtures.json": "3874c3f996c9676549b7a7117ba5219a7a58eee275cbb302fcaede85478ee119", + "static/workspace-projects.js": "4a5e2e4c61cea0cf9f3a30772cf5f47408c72a77d6fe2afe63a2717e83157b7a", + "static/workspace-real.js": "e3a725c5c010afdfa74e337aee991f2ca3cd9c08740a64e0927d106ac263a91a", + "static/workspace-service.js": "9a924f671f604221de3c055a75da2fce3a82666d4090cf50477ab757032228bd", + "static/workspaces-data.css": "9f6151c17baf04b589ab997a3063435b1d035a397a92a48b0badf2911f08091a", + "static/workspaces-data.js": "da2807d18da4ef4c366bb178d30b336d0b9896fd349b76dd97183e7237022dd7", + "static/workspaces.css": "a8152b6da7deb3a8ce65a0de50f0854ed762058954e0015488cc680438bdde60", + "static/workspaces.html": "b68e85cb8d06055506d34c691896915fb7adf62d4c669a3d558299db5e6a0fac", + "static/workspaces.js": "33eeb0ed47e1f6d8b651edc473d44048d8d7b872d581c19960a023a0bd0fda25" + }, + "adapter_exists": false, + "checks": [ + "Normal owner CLI issuance and real HttpOnly session exchange succeeded without adapters.", + "/workspaces served from copied static sources.", + "/workbench served from copied static sources.", + "/workspace-external.js served from copied static sources.", + "/scientific-fields.js served from copied static sources.", + "Generic project hash inspection succeeded.", + "Authored generic scientific bundle imported as draft through production admission.", + "Data record list and exact scientific slice/physical coordinates inspected through production HTTP.", + "Runs list remains empty; no producer or recipe executed.", + "Copied source/static bytes unchanged; optional adapters directory remains absent." + ], + "record_id": "4fd1bc32c5ae4a86a0de6aea3c0e2a5e", + "bundle_sha256": "32d79d5e4c8881fcfd6c858b70e485c61a636ac1e8ea7d4dbd6c52c7bdc47ed7", + "slice_sha256": "ad6bef6b38eafee0614b99c1c0dc1d5752ec61cb2fce67863882d782ecc8b7c1", + "field": "scalar" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/artifact-manifest.json b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/artifact-manifest.json new file mode 100644 index 00000000..a0e0eec1 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/artifact-manifest.json @@ -0,0 +1 @@ +{"artifacts":[{"bytes":1007942,"media_type":"application/octet-stream","name":"result.json","role":"comparison_result","sha256":"d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448"},{"bytes":871725,"media_type":"application/octet-stream","name":"timing.json","role":"comparison_timing","sha256":"0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b"},{"bytes":272149,"media_type":"application/octet-stream","name":"input-model.pt","role":"source_checkpoint","sha256":"25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9"},{"bytes":26712,"media_type":"application/octet-stream","name":"comparison-receipt.json","role":"comparison_receipt","sha256":"d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a"},{"bytes":96833,"media_type":"application/octet-stream","name":"input-freeze.json","role":"source_freeze","sha256":"7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba"},{"bytes":1400,"media_type":"application/octet-stream","name":"comparison-freeze.json","role":"comparison_configuration","sha256":"5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076"}],"custody_receipt_sha256":"89d1062ee70ea2a47eb1006b271280d0e8e48fb567b6b0c52be4dd68ff649318","material_sha256":"abb1b223cbcb86b55a39f6e02d42b6c2cff64d6409d66cb41f2c596ffb254a52","request_sha256":"d13614bb5c8e575817717ed0b11cd1440e710959394f2ba600be3ae9b0adfef1","run_id":"0cc3ef6b5f9e460f873c8eda1ba3d525","schema_version":1} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/custody-receipt.json b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/custody-receipt.json new file mode 100644 index 00000000..5610265f --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/custody-receipt.json @@ -0,0 +1 @@ +{"attempt_id":"0cc3ef6b5f9e460f873c8eda1ba3d525","drained":true,"echild":true,"go_received":true,"helper":{"boot_id":"14f8821d-b20e-4ebd-87a4-df441b737def","pid":359968,"start_ticks":"3579327"},"leader_returncode":0,"outcome":"completed","reason":"exited","request_sha256":"d13614bb5c8e575817717ed0b11cd1440e710959394f2ba600be3ae9b0adfef1","schema_version":1,"spawn_closed":true,"spec_sha256":"1fb60c4106b70ff50f142f8648fa4ed26dd0a8f2858c9c427c1df24e3537801d","streams":{"stderr":{"bytes":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},"stdout":{"bytes":7808,"sha256":"6ebdc94101bf9029934d878a60d764af6465955e87ebfd43f993d7ecddf9364a"}}} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/runtime-proof.json b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/runtime-proof.json new file mode 100644 index 00000000..517bc842 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/runtime-proof.json @@ -0,0 +1,72 @@ +{ + "source": "Explicit owner-local QA read of hash-bound custodian stdout; not HTTP download or agent API parity", + "tuldok_cpu_runtime": { + "environment": { + "CUDA_VISIBLE_DEVICES": "", + "LANG": "C", + "LC_ALL": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHOME": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1" + }, + "mappings": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libgfortran-83c28eba-468e71e5.so.5.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libquadmath-2284e583-a9307bba.so.0.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libscipy_openblas64_-f48b354e.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/_core/_multiarray_umath.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/linalg/_umath_linalg.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_C.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libc10.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libgomp.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libshm.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_cpu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_global_deps.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_python.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/bin/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/ld-linux-x86-64.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libc.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libdl.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libgcc_s.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libm.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libpthread.so.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/librt.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libstdc++.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libutil.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libz.so.1" + ], + "module_paths": { + "ctypes": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/ctypes/__init__.py", + "numpy": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/__init__.py", + "os": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/os.py", + "torch": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/__init__.py" + }, + "packages": { + "numpy": "2.5.4", + "torch": "2.8.0+cpu" + }, + "python": "3.12.14 (main, Aug 25 2026, 14:00:49) [Clang 22.1.3 ]", + "snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7", + "sys_path": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/lib-dynload", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages" + ], + "virtual_modules": { + "torch.classes": { + "class": "_Classes", + "defining_module": "torch._classes", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_classes.py", + "pseudo_file": "_classes.py" + }, + "torch.ops": { + "class": "_Ops", + "defining_module": "torch._ops", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_ops.py", + "pseudo_file": "_ops.py" + } + } + } +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/stderr.log b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/stderr.log new file mode 100644 index 00000000..e69de29b diff --git a/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/stdout.log b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/stdout.log new file mode 100644 index 00000000..342ff7d2 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/owner-local-custody/stdout.log @@ -0,0 +1,31 @@ +{"tuldok_cpu_runtime":{"environment":{"CUDA_VISIBLE_DEVICES":"","LANG":"C","LC_ALL":"C","PYTHONCOERCECLOCALE":"0","PYTHONDONTWRITEBYTECODE":"1","PYTHONHOME":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python","PYTHONNOUSERSITE":"1","PYTHONUTF8":"1"},"mappings":["/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libgfortran-83c28eba-468e71e5.so.5.0.0","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libquadmath-2284e583-a9307bba.so.0.0.0","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libscipy_openblas64_-f48b354e.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/_core/_multiarray_umath.cpython-312-x86_64-linux-gnu.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/linalg/_umath_linalg.cpython-312-x86_64-linux-gnu.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_C.cpython-312-x86_64-linux-gnu.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libc10.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libgomp.so.1","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libshm.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_cpu.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_global_deps.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_python.so","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/bin/python3.12","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/ld-linux-x86-64.so.2","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libc.so.6","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libdl.so.2","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libgcc_s.so.1","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libm.so.6","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libpthread.so.0","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/librt.so.1","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libstdc++.so.6","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libutil.so.1","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libz.so.1"],"module_paths":{"ctypes":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/ctypes/__init__.py","numpy":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/__init__.py","os":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/os.py","torch":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/__init__.py"},"packages":{"numpy":"2.5.4","torch":"2.8.0+cpu"},"python":"3.12.14 (main, Aug 25 2026, 14:00:49) [Clang 22.1.3 ]","snapshot_sha256":"cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7","sys_path":["/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/lib-dynload","/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages"],"virtual_modules":{"torch.classes":{"class":"_Classes","defining_module":"torch._classes","defining_path":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_classes.py","pseudo_file":"_classes.py"},"torch.ops":{"class":"_Ops","defining_module":"torch._ops","defining_path":"/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_ops.py","pseudo_file":"_ops.py"}}}} +CPU jacobi seed 900 repeat 1: 35.797 s +CPU sgs seed 900 repeat 1: 24.930 s +CPU learned seed 900 repeat 1: 4.616 s +CPU sgs seed 901 repeat 1: 25.073 s +CPU learned seed 901 repeat 1: 4.666 s +CPU jacobi seed 901 repeat 1: 34.232 s +CPU learned seed 902 repeat 1: 4.847 s +CPU jacobi seed 902 repeat 1: 36.442 s +CPU sgs seed 902 repeat 1: 26.831 s +CPU jacobi seed 903 repeat 1: 34.917 s +CPU sgs seed 903 repeat 1: 26.386 s +CPU learned seed 903 repeat 1: 4.733 s +CPU accuracy 900 / jacobi: 8.29e-08 +CPU accuracy 900 / sgs: 8.55e-08 +CPU accuracy 901 / jacobi: 8.89e-08 +CPU accuracy 901 / sgs: 9e-08 +CPU accuracy 902 / jacobi: 8.43e-08 +CPU accuracy 902 / sgs: 8.15e-08 +CPU accuracy 903 / jacobi: 8.23e-08 +CPU accuracy 903 / sgs: 8.5e-08 +{ + "median_120_frame_wall_ms": { + "jacobi": 35357.278365998354, + "sgs": 25729.378062998876, + "learned": 4699.447435497859 + }, + "speedup_vs_jacobi": 7.523709723599154, + "speedup_vs_sgs": 5.474979434528584, + "max_relative_residual": 9.987588478738668e-09 +} diff --git a/docs/plans/generic-scientific-projects/reports/p5a/parallel-build.log b/docs/plans/generic-scientific-projects/reports/p5a/parallel-build.log new file mode 100644 index 00000000..96ee64b9 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/parallel-build.log @@ -0,0 +1,9 @@ + Compiling crossbeam-utils v0.8.23 + Compiling crossbeam-epoch v0.9.21 + Compiling crossbeam-deque v0.8.8 + Compiling rayon-core v1.13.0 + Compiling either v1.19.0 + Compiling rayon v1.11.0 + Compiling rheon v0.1.0 (/workspace/scratch/tuldok-science-probe/Rheon) + Compiling rheon-trajectory-parallel v0.1.0 (/workspace/scratch/tuldok-science-probe/lab/probes/trajectory-parallel) + Finished `release` profile [optimized] target(s) in 8.58s diff --git a/docs/plans/generic-scientific-projects/reports/p5a/parallel-fetch.log b/docs/plans/generic-scientific-projects/reports/p5a/parallel-fetch.log new file mode 100644 index 00000000..998d7584 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/parallel-fetch.log @@ -0,0 +1,14 @@ + Updating crates.io index + Downloading crates ... + Downloaded quote v1.0.47 + Downloaded crossbeam-utils v0.8.23 + Downloaded either v1.19.0 + Downloaded crossbeam-epoch v0.9.21 + Downloaded unicode-ident v1.0.26 + Downloaded crossbeam-deque v0.8.8 + Downloaded serde_derive v1.0.229 + Downloaded rayon-core v1.13.0 + Downloaded serde v1.0.229 + Downloaded proc-macro2 v1.0.107 + Downloaded rayon v1.11.0 + Downloaded syn v3.0.6 diff --git a/docs/plans/generic-scientific-projects/reports/p5a/parallel-receipt.json b/docs/plans/generic-scientific-projects/reports/p5a/parallel-receipt.json new file mode 100644 index 00000000..a271f384 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/parallel-receipt.json @@ -0,0 +1,262 @@ +{ + "schema_version": 1, + "started_utc": "2026-10-13T18:48:03.531591+00:00", + "sources": { + "lab": { + "commit": "e8a90bfef6b2955c502008ccc62443e29dd9f2ec", + "tree": "deb79c7c491768a22427bb77198b3b670ddc8be1", + "status_before": "", + "status_after": "" + }, + "Rheon": { + "commit": "2209012bc5f902693d0f695ec1c2d854126af317", + "tree": "e30b7135eabad4827ee6e21c75715af0f5408d78", + "status_before": "", + "status_after": "" + } + }, + "lock_sha256_before": "6e75b61a8b16dce290081b47c9d0cdb9d21a96c4d53b0385898bc05001307e98", + "toolchain_archive_sha256": "d2ccef59dd9f7439f2c694948069f789a044dc1addcc0803613232af8f88ee0c", + "toolchain": { + "rustc": "rustc 1.92.0 (ded5c06cf 2025-12-08)", + "cargo": "cargo 1.92.0 (344c4567c 2025-10-21)" + }, + "commands": [ + { + "argv": [ + "cargo", + "fetch", + "--locked", + "--manifest-path", + "lab/probes/trajectory-parallel/Cargo.toml" + ], + "exit_code": 0, + "log": "fetch.log", + "log_sha256": "026e5ca76bb708ced64deeeb5ffe4a7a4123f1f1bc94fda5d85cc113b2860895" + }, + { + "argv": [ + "cargo", + "build", + "--release", + "--locked", + "--offline", + "--manifest-path", + "lab/probes/trajectory-parallel/Cargo.toml" + ], + "exit_code": 0, + "log": "build.log", + "log_sha256": "f4b595fb99cb710814f05b0dd39467f53758dee814f9bdc46b5c5a83eb3eba7f" + } + ], + "archives": [ + { + "name": "anyhow", + "version": "1.0.100", + "expected_sha256": "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.100.crate", + "sha256": "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + } + ] + }, + { + "name": "crossbeam-deque", + "version": "0.8.8", + "expected_sha256": "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/crossbeam-deque-0.8.8.crate", + "sha256": "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" + } + ] + }, + { + "name": "crossbeam-epoch", + "version": "0.9.21", + "expected_sha256": "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/crossbeam-epoch-0.9.21.crate", + "sha256": "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" + } + ] + }, + { + "name": "crossbeam-utils", + "version": "0.8.23", + "expected_sha256": "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/crossbeam-utils-0.8.23.crate", + "sha256": "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + } + ] + }, + { + "name": "either", + "version": "1.19.0", + "expected_sha256": "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/either-1.19.0.crate", + "sha256": "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + } + ] + }, + { + "name": "itoa", + "version": "1.0.18", + "expected_sha256": "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/itoa-1.0.18.crate", + "sha256": "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + } + ] + }, + { + "name": "memchr", + "version": "2.8.3", + "expected_sha256": "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/memchr-2.8.3.crate", + "sha256": "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + } + ] + }, + { + "name": "proc-macro2", + "version": "1.0.107", + "expected_sha256": "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/proc-macro2-1.0.107.crate", + "sha256": "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" + } + ] + }, + { + "name": "quote", + "version": "1.0.47", + "expected_sha256": "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/quote-1.0.47.crate", + "sha256": "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" + } + ] + }, + { + "name": "rayon", + "version": "1.11.0", + "expected_sha256": "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/rayon-1.11.0.crate", + "sha256": "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" + } + ] + }, + { + "name": "rayon-core", + "version": "1.13.0", + "expected_sha256": "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/rayon-core-1.13.0.crate", + "sha256": "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" + } + ] + }, + { + "name": "serde", + "version": "1.0.229", + "expected_sha256": "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/serde-1.0.229.crate", + "sha256": "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" + } + ] + }, + { + "name": "serde_core", + "version": "1.0.229", + "expected_sha256": "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/serde_core-1.0.229.crate", + "sha256": "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" + } + ] + }, + { + "name": "serde_derive", + "version": "1.0.229", + "expected_sha256": "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/serde_derive-1.0.229.crate", + "sha256": "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" + } + ] + }, + { + "name": "serde_json", + "version": "1.0.151", + "expected_sha256": "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/serde_json-1.0.151.crate", + "sha256": "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" + } + ] + }, + { + "name": "syn", + "version": "3.0.6", + "expected_sha256": "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/syn-3.0.6.crate", + "sha256": "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" + } + ] + }, + { + "name": "unicode-ident", + "version": "1.0.26", + "expected_sha256": "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/unicode-ident-1.0.26.crate", + "sha256": "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + } + ] + }, + { + "name": "zmij", + "version": "1.0.23", + "expected_sha256": "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b", + "archives": [ + { + "path": "cargo-home/registry/cache/index.crates.io-1949cf8c6b5b557f/zmij-1.0.23.crate", + "sha256": "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + } + ] + } + ], + "all_locked_archives_verified": true, + "lock_sha256_after": "6e75b61a8b16dce290081b47c9d0cdb9d21a96c4d53b0385898bc05001307e98", + "status": "built", + "binary": { + "path": "/workspace/scratch/tuldok-science-probe/target/release/rheon-trajectory-parallel", + "bytes": 1063328, + "sha256": "583c939548d68ec7b25cf60bb9434ca1416f28ab6c06b3cad68a1aaf10b344db" + }, + "native_elf_dynamic": "\nDynamic section at offset 0xbedc8 contains 30 entries:\n Tag Type Name/Value\n 0x0000000000000001 (NEEDED) Shared library: [libgcc_s.so.1]\n 0x0000000000000001 (NEEDED) Shared library: [libm.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [libc.so.6]\n 0x0000000000000001 (NEEDED) Shared library: [ld-linux-x86-64.so.2]\n 0x000000000000001e (FLAGS) BIND_NOW\n 0x000000006ffffffb (FLAGS_1) Flags: NOW PIE\n 0x0000000000000015 (DEBUG) 0x0\n 0x0000000000000007 (RELA) 0x12a8\n 0x0000000000000008 (RELASZ) 43992 (bytes)\n 0x0000000000000009 (RELAENT) 24 (bytes)\n 0x000000006ffffff9 (RELACOUNT) 1750\n 0x0000000000000017 (JMPREL) 0xbe80\n 0x0000000000000002 (PLTRELSZ) 48 (bytes)\n 0x0000000000000003 (PLTGOT) 0xc0e68\n 0x0000000000000014 (PLTREL) RELA\n 0x0000000000000006 (SYMTAB) 0x340\n 0x000000000000000b (SYMENT) 24 (bytes)\n 0x0000000000000005 (STRTAB) 0xd7c\n 0x000000000000000a (STRSZ) 1317 (bytes)\n 0x000000006ffffef5 (GNU_HASH) 0xd60\n 0x0000000000000019 (INIT_ARRAY) 0xba028\n 0x000000000000001b (INIT_ARRAYSZ) 16 (bytes)\n 0x000000000000001a (FINI_ARRAY) 0xba020\n 0x000000000000001c (FINI_ARRAYSZ) 8 (bytes)\n 0x000000000000000c (INIT) 0xb9fa8\n 0x000000000000000d (FINI) 0xb9fc0\n 0x000000006ffffff0 (VERSYM) 0xb50\n 0x000000006ffffffe (VERNEED) 0xbfc\n 0x000000006fffffff (VERNEEDNUM) 4\n 0x0000000000000000 (NULL) 0x0\n", + "finished_utc": "2026-10-13T18:48:17.014091+00:00" +} diff --git a/docs/plans/generic-scientific-projects/reports/p5a/runtime-qualification.json b/docs/plans/generic-scientific-projects/reports/p5a/runtime-qualification.json new file mode 100644 index 00000000..013ff35f --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/runtime-qualification.json @@ -0,0 +1,161 @@ +{ + "cancel": { + "custody_sha256": "10169614d7e92977f174be4d5183d89673cc7e0f34019b6f967e04dfaa9119d4", + "run_id": "a8298e4e255a46d4beef0ec9f73e38d0", + "runtime": { + "environment": { + "CUDA_VISIBLE_DEVICES": "", + "LANG": "C", + "LC_ALL": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHOME": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1" + }, + "mappings": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libgfortran-83c28eba-468e71e5.so.5.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libquadmath-2284e583-a9307bba.so.0.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libscipy_openblas64_-f48b354e.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/_core/_multiarray_umath.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/linalg/_umath_linalg.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_C.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libc10.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libgomp.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libshm.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_cpu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_global_deps.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_python.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/bin/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/ld-linux-x86-64.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libc.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libdl.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libgcc_s.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libm.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libpthread.so.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/librt.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libstdc++.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libutil.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libz.so.1" + ], + "module_paths": { + "ctypes": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/ctypes/__init__.py", + "numpy": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/__init__.py", + "os": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/os.py", + "torch": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/__init__.py" + }, + "packages": { + "numpy": "2.5.4", + "torch": "2.8.0+cpu" + }, + "python": "3.12.14 (main, Aug 25 2026, 14:00:49) [Clang 22.1.3 ]", + "snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7", + "sys_path": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/lib-dynload", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages" + ], + "virtual_modules": { + "torch.classes": { + "class": "_Classes", + "defining_module": "torch._classes", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_classes.py", + "pseudo_file": "_classes.py" + }, + "torch.ops": { + "class": "_Ops", + "defining_module": "torch._ops", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_ops.py", + "pseudo_file": "_ops.py" + } + } + }, + "state": "cancelled" + }, + "checks": [ + "Actual CPU forward/backward, exact NumPy bytes and weights_only tensor checkpoint reload completed behind approved production custody.", + "Actual initialized captured CPU child cancelled with confirmed drain and no published artifacts.", + "Catalog reopen preserves exact checkpoint, NumPy and result artifact bytes." + ], + "compute": { + "custody_sha256": "e6c2876fb0faf91b7abb02ef7f27ae63afae9818a7d40790c43e179405d42586", + "run_id": "44272474e32544898feb39f651d997b9", + "runtime": { + "environment": { + "CUDA_VISIBLE_DEVICES": "", + "LANG": "C", + "LC_ALL": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHOME": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python", + "PYTHONNOUSERSITE": "1", + "PYTHONUTF8": "1" + }, + "mappings": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libgfortran-83c28eba-468e71e5.so.5.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libquadmath-2284e583-a9307bba.so.0.0.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy.libs/libscipy_openblas64_-f48b354e.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/_core/_multiarray_umath.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/linalg/_umath_linalg.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_C.cpython-312-x86_64-linux-gnu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libc10.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libgomp.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libshm.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_cpu.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_global_deps.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/lib/libtorch_python.so", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/bin/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/ld-linux-x86-64.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libc.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libdl.so.2", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libgcc_s.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libm.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libpthread.so.0", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/librt.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libstdc++.so.6", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libutil.so.1", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/system/lib/libz.so.1" + ], + "module_paths": { + "ctypes": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/ctypes/__init__.py", + "numpy": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/numpy/__init__.py", + "os": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/os.py", + "torch": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/__init__.py" + }, + "packages": { + "numpy": "2.5.4", + "torch": "2.8.0+cpu" + }, + "python": "3.12.14 (main, Aug 25 2026, 14:00:49) [Clang 22.1.3 ]", + "snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7", + "sys_path": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/python/lib/python3.12/lib-dynload", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages" + ], + "virtual_modules": { + "torch.classes": { + "class": "_Classes", + "defining_module": "torch._classes", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_classes.py", + "pseudo_file": "_classes.py" + }, + "torch.ops": { + "class": "_Ops", + "defining_module": "torch._ops", + "defining_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/runtime-snapshots/cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7/packages/torch/_ops.py", + "pseudo_file": "_ops.py" + } + } + }, + "state": "completed" + }, + "installed_runtime_modified": false, + "original_runtime_filesystem_absence_tested": false, + "probe_source_sha256": "749469dab68f24c9d8dbff5652b3addb8b6773ceccf4960d7e56163e84055a7a", + "project_id": "d26f1f282e1d4fd9a07a0ed5f49a49cc", + "scope": "Production approved Runs with captured CPU snapshot; authored computation, not historical model/science qualification", + "snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5a/source-files.json b/docs/plans/generic-scientific-projects/reports/p5a/source-files.json new file mode 100644 index 00000000..f7f1f73b --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5a/source-files.json @@ -0,0 +1,70 @@ +{ + "access.py": { + "bytes": 27243, + "sha256": "50681dd5b2aaba01a311a7465b15161ca264fa4ca9812b95f15914adb5d808f4" + }, + "runtime_snapshots.py": { + "bytes": 22094, + "sha256": "dad9e4ddf9c11c0e1f2d5b9a7afd6877e7be375fe02030c65b684bca2b0ef45e" + }, + "recipe_contract.py": { + "bytes": 40379, + "sha256": "c59180b3468e9a4891a7dfae9676f1d77a6a9af461260dd9020d042251292c50" + }, + "recipe_process.py": { + "bytes": 48820, + "sha256": "72f3e292305cedeff136eb2a2592668113eaeb2c1c658d9933ca85865eb8fa39" + }, + "runs.py": { + "bytes": 80981, + "sha256": "a6c1d9f91503179e1e722d17ed7c9c3b0b9e9540e4afb6ee979d177c10235562" + }, + "static/workspace-external.js": { + "bytes": 21140, + "sha256": "a28436e15312f5f67203dc6c2d2c618421b46ace42bd32cad4000df3794c06d2" + }, + "tests/browser_workspace_cpu.cjs": { + "bytes": 17628, + "sha256": "6911c8b4f568cedcd9691fbe53d2837ba344fb08f423406e9ddaf4114457557c" + }, + "tests/qualify_captured_cpu_runtime.py": { + "bytes": 14113, + "sha256": "36198b4d7935d31997e4ec5ed48b96a1aaefd4f284c1f737e1b00c840cd386e2" + }, + "tests/qualify_cpu_comparison.py": { + "bytes": 9225, + "sha256": "ccf1d67354b3898fe6eb2f142c5c5e7665318f218df193259d43843366b436ed" + }, + "tests/test_cpu_recipe_contract.py": { + "bytes": 6583, + "sha256": "1a45b1e6f4f58d3a2002686aed4f04afc7f8736cd618641769bb5e6f8c762ec5" + }, + "tests/test_runtime_snapshots.py": { + "bytes": 9566, + "sha256": "4a932759b55d641a6aeb329430b43460c0f1afea3a4d3cd2a3f1c3051ab376d0" + }, + "tests/test_access.py": { + "bytes": 13230, + "sha256": "31a480dfec02e54db48f7a324714a6b101c34193129a71827bcd61e725d813ea" + }, + "tests/test_recipe_process.py": { + "bytes": 23976, + "sha256": "a3884c8af0f50c3a69238f7b8a8a4a7e84ea67a8fbc75772e8b1a2da3059a9bb" + }, + "tests/test_rheon_cpu_adapter.py": { + "bytes": 11048, + "sha256": "5e278f8d0060b9d1009ac89650a7f7ce635adb9cdae5a82960f1d3360a79b1f7" + }, + "adapters/rheon/cpu_compare.py": { + "bytes": 11992, + "sha256": "e2e14eb6f354b88a0123bcdd710d4f990102efb97f7cdb584db4700e7f05ab97" + }, + "adapters/rheon/cpu_project.py": { + "bytes": 5317, + "sha256": "fe22a72b6dfc1b703c5407ba5691cef2117f6d5adeddf0d504f5c94270c003cf" + }, + "adapters/rheon/native_transport.py": { + "bytes": 3794, + "sha256": "642bfab52028053561d20c4a6522bf131cb452ef4f69e55efa0786324c0c9d55" + } +} diff --git a/recipe_contract.py b/recipe_contract.py index d715ba16..d83851d4 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -19,6 +19,9 @@ TYPES = ('input', 'execute', 'output') PORTS = {'input': (None, 'data'), 'execute': ('data', 'artifacts'), 'output': ('artifacts', None)} TRUST = 'Trusted local code; filesystem and network are not sandboxed. Runtime installation must remain immutable for the attempt.' +CPU_PROFILE = 'captured_cpu_python_v1' +CPU_DESCRIPTOR = 'runtime-bootstrap/snapshot.json' +CPU_BOOTSTRAP = 'runtime-bootstrap/recipe_process.py' HOST_NAMES = ('recipe_contract.py', 'projects.py', 'pipeline.py', 'recipe_process.py') IMPORTED_HOST_FILES = {name: hashlib.sha256((Path(__file__).resolve().parent/name).read_bytes()).hexdigest() for name in HOST_NAMES} @@ -48,15 +51,21 @@ def decode_definition(raw): raise WorkbenchError('Recipe definition exceeds 64 KiB.') value = decode(raw) closed(value, ('schema_version', 'format', 'id', 'name', 'version', 'runtime', 'sources', 'entrypoint', - 'arguments', 'parameters', 'inputs', 'outputs', 'environment', 'resources'), ('native_sources',)) + 'arguments', 'parameters', 'inputs', 'outputs', 'environment', 'resources'), ('native_sources','runtime_snapshot_sha256')) if type(value['schema_version']) is not int or value['schema_version'] != 1 or value['format'] != 'external_recipe_v1': raise WorkbenchError('Only external_recipe_v1 is supported.', 'unsupported', 400) named(value['id']) bounded_int(value['version'], 1, 1, 'Recipe version') if not isinstance(value['name'], str) or not 1 <= len(value['name']) <= 120: raise WorkbenchError('Supply a bounded recipe name.') - if value['runtime'] != 'system_python': - raise WorkbenchError('Only the configured system_python runtime is qualified.', 'unsupported', 400) + if value['runtime'] not in ('system_python', CPU_PROFILE): + raise WorkbenchError('Unsupported recipe runtime profile.', 'unsupported', 400) + if value['runtime'] == CPU_PROFILE: + snapshot = value.get('runtime_snapshot_sha256') + if not isinstance(snapshot, str) or not SHA.fullmatch(snapshot): + raise WorkbenchError('Captured CPU recipes require a runtime snapshot SHA256.') + elif 'runtime_snapshot_sha256' in value: + raise WorkbenchError('System Python recipes cannot select a CPU snapshot.') sources = value['sources'] if not isinstance(sources, list) or not 1 <= len(sources) <= 64: raise WorkbenchError('Declare 1–64 source files.') @@ -98,8 +107,9 @@ def decode_definition(raw): for key, rule in value['inputs'].items(): named(key); closed(rule, ('max_bytes',)) bounded_int(rule['max_bytes'], 1, 40 * 1024 * 1024, 'Input bytes') - if not isinstance(value['arguments'], list) or len(value['arguments']) > 59: - raise WorkbenchError('Declare at most 59 argument tokens plus the fixed runtime entrypoint.') + argument_limit = 56 if value['runtime'] == CPU_PROFILE else 59 + if not isinstance(value['arguments'], list) or len(value['arguments']) > argument_limit: + raise WorkbenchError('Recipe arguments exceed the runtime argument bound.') for token in value['arguments']: if not isinstance(token, dict) or len(token) != 1: raise WorkbenchError('Arguments are single typed tokens.') @@ -442,6 +452,16 @@ def prepare(projects, graph, execution_sha256, profile=None): captured[native_policy_path(source)] = encode(policy).encode() if len(captured) > MAX_FILES or sum(map(len,captured.values())) > MAX_CAPTURE_BYTES: raise WorkbenchError('Generated native policies exceed the capture bound.') + cpu_snapshot = None + if definition['runtime'] == CPU_PROFILE: + import recipe_process + from runtime_snapshots import RuntimeSnapshots + cpu_snapshot = RuntimeSnapshots(projects.db, projects.access, projects.db.path).get( + definition['runtime_snapshot_sha256'], verify=True) + captured[CPU_DESCRIPTOR] = recipe_process.encoded(cpu_snapshot) + captured[CPU_BOOTSTRAP] = recipe_process.SOURCE + if len(captured) > MAX_FILES or sum(map(len,captured.values())) > MAX_CAPTURE_BYTES: + raise WorkbenchError('Generated CPU bootstrap exceeds the capture bound.') host = host_identity() material = {'schema_version': 1, 'catalog_id': projects.db.dataset_id, 'project': inspected['receipt'], 'graph_execution_sha256': execution_sha256, 'recipe': definition, 'parameters': values, 'inputs': inputs, @@ -449,6 +469,7 @@ def prepare(projects, graph, execution_sha256, profile=None): 'runtime': runtime, 'host_files': host, 'permissions': {'filesystem':'trusted-local', 'network':'not_isolated'}, 'environment': {'LANG':'C.UTF-8'}, 'resources': definition['resources']} if policies is not None: material['native_policies'] = policies + if cpu_snapshot is not None: material['cpu_snapshot'] = cpu_snapshot validate_material(material) return {'valid':True, 'status':'valid', 'material':material, 'material_sha256':digest(material), 'validation_token':digest(material), 'parameter_definitions':definition['parameters'], 'outputs':definition['outputs'], 'runtime':runtime, @@ -458,7 +479,7 @@ def prepare(projects, graph, execution_sha256, profile=None): def validate_material(material): """Cold structural custody checks; no filesystem access or current-code comparison.""" closed(material, ('schema_version', 'catalog_id', 'project', 'graph_execution_sha256', 'recipe', - 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources'), ('native_policies',)) + 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources'), ('native_policies','cpu_snapshot')) if type(material['schema_version']) is not int or material['schema_version'] != 1 or not isinstance(material['catalog_id'], str) or not material['catalog_id']: raise WorkbenchError('Invalid execution material identity.') def sha(value): @@ -482,6 +503,23 @@ def receipt(value): raise WorkbenchError('Input receipt belongs to another project snapshot.') expected['input/'+name] = item['measured_sha256'] generated = validate_native_policies(material,definition) + if definition['runtime'] == CPU_PROFILE: + import recipe_process + try: + descriptor = recipe_process.validate_cpu_snapshot_descriptor(material.get('cpu_snapshot')) + except recipe_process.ProcessError as error: + raise WorkbenchError(str(error)) from None + if descriptor['catalog_id'] != material['catalog_id'] or descriptor['manifest_sha256'] != definition['runtime_snapshot_sha256']: + raise WorkbenchError('CPU snapshot differs from the approved catalog and recipe.') + generated[CPU_DESCRIPTOR] = recipe_process.encoded(descriptor) + # Historical bootstrap identity belongs to retained host material, never + # to the currently installed helper source during a cold read. + if not isinstance(material['host_files'],dict): + raise WorkbenchError('Invalid retained host closure.') + expected[CPU_BOOTSTRAP] = material['host_files'].get('recipe_process.py') + sha(expected[CPU_BOOTSTRAP]) + elif 'cpu_snapshot' in material: + raise WorkbenchError('System Python material cannot contain a CPU snapshot.') expected.update({path:file_digest(raw) for path,raw in generated.items()}) captures = material['captures'] if not isinstance(captures, dict) or set(captures) != set(expected) or len(captures) > MAX_FILES: @@ -489,7 +527,7 @@ def receipt(value): total = 0 for path, item in captures.items(): relative_path(path); closed(item, ('bytes','sha256')); sha(item['sha256']) - limit = MAX_SOURCE_BYTES if path.startswith(('source/','native-policy/')) else definition['inputs'][path.split('/',1)[1]]['max_bytes'] + limit = MAX_SOURCE_BYTES if path.startswith(('source/','native-policy/','runtime-bootstrap/')) else definition['inputs'][path.split('/',1)[1]]['max_bytes'] bounded_int(item['bytes'],0,limit,'Capture bytes'); total += item['bytes'] if path in generated and item['bytes'] != len(generated[path]): raise WorkbenchError('Generated native policy size differs.') if item['sha256'] != expected[path]: raise WorkbenchError('Captured digest contradicts source/input identity.') @@ -522,6 +560,9 @@ def spec_for_material(material, staging): definition = material['recipe']; runtime = material['runtime'] records = [dict(path=str(staging/relative), **record) for relative,record in sorted(material['captures'].items())] argv = [runtime['executable'], *runtime['flags'], str(staging/'source'/definition['entrypoint'])] + if definition['runtime'] == CPU_PROFILE: + argv = [runtime['executable'], *runtime['flags'], str(staging/CPU_BOOTSTRAP), + '--cpu-launch', str(staging/CPU_DESCRIPTOR), str(staging/'source'/definition['entrypoint'])] for token in definition['arguments']: kind, value = next(iter(token.items())) if kind == 'literal': arg = value @@ -537,7 +578,8 @@ def spec_for_material(material, staging): 'max_output_bytes':definition['resources']['max_output_bytes'], 'output_dir':str(output),'captured_files':records, 'runtime_files':[dict(path=path,**record) for path,record in sorted(runtime['files'].items())], 'trusted_install_owner_uid':runtime['trusted_install_owner_uid']} - if 'native_policies' in material: spec['native_loader_policy'] = NATIVE_POLICY + if 'native_policies' in material or 'cpu_snapshot' in material: spec['native_loader_policy'] = NATIVE_POLICY + if 'cpu_snapshot' in material: spec['cpu_snapshot'] = material['cpu_snapshot'] return spec diff --git a/recipe_process.py b/recipe_process.py index 96aeb92a..ddf73db1 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -69,10 +69,236 @@ def absolute(value): return text(value) and Path(value).is_absolute() and '..' not in Path(value).parts +CPU_PROFILE = 'captured_cpu_python_v1' +CPU_MANIFEST_LIMIT = 16 * 1024 * 1024 +CPU_FILE_LIMIT = 512 * 1024 * 1024 +CPU_TOTAL_LIMIT = 2 * 1024 * 1024 * 1024 +CPU_FILE_COUNT = 40000 +CPU_READY_SECONDS = 60 + + +def _cpu_relative(value): + return (text(value, 1024) and not value.startswith('/') and '\\' not in value + and len(value.split('/')) <= 32 + and all(part not in ('', '.', '..') for part in value.split('/'))) + + +def _cpu_version_packages(value): + if (type(value) is not dict or not {'numpy', 'torch'} <= set(value) + or len(value) > 256 or any(not text(k, 128) or not text(v, 128) for k, v in value.items())): + raise ProcessError('Invalid captured CPU package inventory.') + + +def _cpu_launch(value): + keys = {'python', 'loader', 'home', 'stdlib', 'dynload', 'packages', 'library_dirs'} + if type(value) is not dict or set(value) != keys: + raise ProcessError('Invalid captured CPU launch layout.') + if any(not _cpu_relative(value[key]) for key in keys - {'library_dirs'}): + raise ProcessError('CPU launch paths must be contained relative paths.') + dirs = value['library_dirs'] + if (type(dirs) is not list or not 1 <= len(dirs) <= 32 + or not all(_cpu_relative(item) and ':' not in item and '$' not in item for item in dirs) + or len(set(dirs)) != len(dirs)): + raise ProcessError('Invalid captured loader search paths.') + + +def validate_cpu_snapshot_descriptor(value): + """Pure bounded descriptor validation; cold receipts need no live runtime.""" + keys = {'schema_version', 'profile', 'catalog_id', 'root', 'manifest_sha256', + 'manifest_bytes', 'file_count', 'total_bytes', 'python_version', 'packages', 'launch'} + if (type(value) is not dict or set(value) != keys or type(value['schema_version']) is not int + or value['schema_version'] != 1 or value['profile'] != CPU_PROFILE): + raise ProcessError('Unsupported captured CPU descriptor.') + if type(value['catalog_id']) is not str or not ID.fullmatch(value['catalog_id']): + raise ProcessError('Invalid captured CPU catalog identity.') + if type(value['manifest_sha256']) is not str or not HASH.fullmatch(value['manifest_sha256']): + raise ProcessError('Invalid captured CPU manifest identity.') + root = value['root'] + if (not absolute(root) or ':' in root or '$' in root or str(Path(root)) != root or Path(root).name != value['manifest_sha256'] + or Path(root).parent.name != 'runtime-snapshots'): + raise ProcessError('CPU snapshot root must bind its manifest address.') + for key, limit in [('manifest_bytes', CPU_MANIFEST_LIMIT), ('file_count', CPU_FILE_COUNT), ('total_bytes', CPU_TOTAL_LIMIT)]: + if type(value[key]) is not int or not 1 <= value[key] <= limit: + raise ProcessError('Captured CPU descriptor exceeds its bound.') + if value['python_version'] != '3.12': + raise ProcessError('Unsupported captured Python version.') + _cpu_version_packages(value['packages']); _cpu_launch(value['launch']) + return decode(encoded(value)) + + +def validate_cpu_manifest(value): + """One canonical manifest authority shared by capture and launch.""" + keys = {'schema_version', 'profile', 'python_version', 'packages', 'launch', 'files', 'total_bytes'} + if (type(value) is not dict or set(value) != keys or type(value['schema_version']) is not int + or value['schema_version'] != 1 or value['profile'] != CPU_PROFILE or value['python_version'] != '3.12'): + raise ProcessError('Unsupported captured CPU manifest.') + _cpu_version_packages(value['packages']); _cpu_launch(value['launch']) + files = value['files'] + if type(files) is not list or not 1 <= len(files) <= CPU_FILE_COUNT: + raise ProcessError('Captured CPU file count exceeds its bound.') + seen = {}; total = 0 + for item in files: + if type(item) is not dict or set(item) != {'path', 'bytes', 'sha256', 'mode', 'kind'}: + raise ProcessError('Invalid captured CPU file record.') + path = item['path'] + if not _cpu_relative(path) or path == 'manifest.json' or path in seen: + raise ProcessError('Duplicate or escaping captured CPU file.') + if (type(item['bytes']) is not int or not 0 <= item['bytes'] <= CPU_FILE_LIMIT + or type(item['sha256']) is not str or not HASH.fullmatch(item['sha256']) + or type(item['mode']) is not int or item['mode'] not in (0o400, 0o500) + or item['kind'] not in ('elf', 'data')): + raise ProcessError('Invalid captured CPU file bounds, hash, mode or kind.') + total += item['bytes']; seen[path] = item + if type(value['total_bytes']) is not int or total != value['total_bytes'] or not 1 <= total <= CPU_TOTAL_LIMIT: + raise ProcessError('Captured CPU total bytes differ or exceed bound.') + if list(seen) != sorted(seen): + raise ProcessError('Captured CPU inventory must be canonically ordered.') + directories = set() + for path in seen: + parents = {str(parent) for parent in Path(path).parents if str(parent) != '.'} + if any(parent in seen for parent in parents): + raise ProcessError('Captured CPU file conflicts with directory path.') + directories.update(parents) + if len(directories) > CPU_FILE_COUNT: + raise ProcessError('Captured CPU directory count exceeds its bound.') + for key in ('python', 'loader'): + item = seen.get(value['launch'][key]) + if not item or item['kind'] != 'elf' or item['mode'] != 0o500: + raise ProcessError('Captured Python and loader must be executable ELF files.') + for relative in [value['launch'][key] for key in ('home', 'stdlib', 'dynload', 'packages')] + value['launch']['library_dirs']: + if relative in seen or not any(path.startswith(relative + '/') for path in seen): + raise ProcessError('Captured CPU launch directory is absent.') + if len(encoded(value)) > CPU_MANIFEST_LIMIT: + raise ProcessError('Captured CPU manifest exceeds its byte bound.') + return decode(encoded(value)) + + +def _cpu_open_root(root): + """Walk every absolute component without following symlinks.""" + fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY) + try: + for part in Path(root).parts[1:]: + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd); fd = next_fd + info = os.fstat(fd) + if info.st_uid != os.getuid() or info.st_mode & 0o022: + raise ProcessError('Captured CPU root is not privately owned.', 'unavailable') + return fd + except BaseException: + os.close(fd); raise + + +def _cpu_open_file(root_fd, relative): + fd = os.dup(root_fd) + try: + parts = relative.split('/') + for part in parts[:-1]: + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd); fd = next_fd + return os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + finally: + os.close(fd) + + +def _cpu_check_deadline(deadline, checkpoint=None): + if checkpoint is not None: checkpoint() + if time.monotonic() > deadline: + raise ProcessError('CPU snapshot preparation deadline exceeded.', 'unavailable') + + +def _cpu_stream_file(root_fd, item, deadline, checkpoint=None): + fd = _cpu_open_file(root_fd, item['path']) + with os.fdopen(fd, 'rb') as handle: + before = os.fstat(handle.fileno()) + if (not stat.S_ISREG(before.st_mode) or before.st_uid != os.getuid() + or before.st_nlink != 1 or stat.S_IMODE(before.st_mode) != item['mode'] or before.st_size != item['bytes']): + raise ProcessError('Captured CPU file size, ownership or mode changed.', 'stale') + hashed = hashlib.sha256(); count = 0; magic = None + while True: + _cpu_check_deadline(deadline, checkpoint) + block = handle.read(1024 * 1024) + if not block: break + if magic is None: magic = block[:4] + count += len(block) + if count > item['bytes']: raise ProcessError('CPU file grew during verification.', 'stale') + hashed.update(block) + after = os.fstat(handle.fileno()) + identity_keys = ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns', 'st_mode', 'st_uid', 'st_nlink') + if (any(getattr(before, key) != getattr(after, key) for key in identity_keys) + or count != item['bytes'] or hashed.hexdigest() != item['sha256'] + or (magic == b'\x7fELF') != (item['kind'] == 'elf')): + raise ProcessError('Captured CPU file identity changed.', 'stale') + + +def verify_cpu_snapshot(descriptor, *, checkpoint=None): + """Stream exact immutable snapshot bytes; never import its packages here.""" + deadline = time.monotonic() + CPU_READY_SECONDS + descriptor = validate_cpu_snapshot_descriptor(descriptor) + try: + root_fd = _cpu_open_root(descriptor['root']) + except OSError as error: + raise ProcessError('Captured CPU root is missing or unsafe.', 'unavailable') from error + try: + fd = _cpu_open_file(root_fd, 'manifest.json') + with os.fdopen(fd, 'rb') as handle: + info = os.fstat(handle.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_size != descriptor['manifest_bytes'] + or info.st_uid != os.getuid() or info.st_nlink != 1 or stat.S_IMODE(info.st_mode) != 0o400): + raise ProcessError('Captured CPU manifest file changed.', 'stale') + raw = handle.read(CPU_MANIFEST_LIMIT + 1) + if len(raw) != descriptor['manifest_bytes'] or digest(raw) != descriptor['manifest_sha256']: + raise ProcessError('Captured CPU manifest hash changed.', 'stale') + manifest = validate_cpu_manifest(decode(raw)) + if raw != encoded(manifest): raise ProcessError('CPU manifest is not canonical.', 'invalid') + for key in ('profile', 'python_version', 'packages', 'launch', 'total_bytes'): + if manifest[key] != descriptor[key]: raise ProcessError('CPU descriptor differs from manifest.', 'stale') + if len(manifest['files']) != descriptor['file_count']: + raise ProcessError('CPU descriptor file count differs.', 'stale') + expected = {item['path'] for item in manifest['files']} + expected_dirs = {str(parent) for path in expected for parent in Path(path).parents if str(parent) != '.'} + observed = set(); directory_count = 0 + pending = [''] + while pending: + _cpu_check_deadline(deadline, checkpoint) + directory = pending.pop(); walk_fd = os.dup(root_fd) + try: + for part in directory.split('/') if directory else []: + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=walk_fd) + os.close(walk_fd); walk_fd = next_fd + with os.scandir(walk_fd) as entries: + for entry in entries: + _cpu_check_deadline(deadline, checkpoint) + relative = str(Path(directory) / entry.name) + info = entry.stat(follow_symlinks=False) + if stat.S_ISDIR(info.st_mode): + directory_count += 1 + if (directory_count > CPU_FILE_COUNT or relative not in expected_dirs + or info.st_uid != os.getuid() or info.st_mode & 0o022): + raise ProcessError('CPU snapshot contains unsafe or unlisted directories.', 'stale') + pending.append(relative) + elif stat.S_ISREG(info.st_mode): + if relative == 'manifest.json': continue + if relative not in expected: + raise ProcessError('CPU snapshot contains unlisted files.', 'stale') + observed.add(relative) + else: + raise ProcessError('CPU snapshot contains symlink or special file.', 'stale') + finally: + os.close(walk_fd) + if observed != expected: raise ProcessError('CPU snapshot file inventory changed.', 'stale') + for item in manifest['files']: _cpu_stream_file(root_fd, item, deadline, checkpoint) + _cpu_check_deadline(deadline, checkpoint) + return manifest + except OSError as error: + raise ProcessError('Captured CPU snapshot is missing, inaccessible or changed.', 'unavailable') from error + finally: + os.close(root_fd) + + def validate_spec(value): keys = {'schema_version','attempt_id','request_sha256','argv','env','cwd','stdin_base64', 'wall_seconds','executable_sha256','max_output_bytes','output_dir','captured_files','runtime_files','trusted_install_owner_uid'} - if type(value) is not dict or set(value) - {'native_loader_policy'} != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: + if type(value) is not dict or set(value) - {'native_loader_policy', 'cpu_snapshot'} != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: raise ProcessError('Unsupported custody spec.') if not isinstance(value['attempt_id'], str) or not ID.fullmatch(value['attempt_id']): raise ProcessError('Invalid attempt identity.') @@ -118,6 +344,20 @@ def validate_spec(value): raise ProcessError('Invalid captured file size.') if sum(item['bytes'] for item in files)>64*1024*1024 or sum(item['bytes'] for item in runtime)>512*1024*1024: raise ProcessError('Runtime/source closure exceeds its byte bound.') + if 'cpu_snapshot' in value: + descriptor = validate_cpu_snapshot_descriptor(value['cpu_snapshot']) + if 'native_loader_policy' not in value: + raise ProcessError('CPU snapshot requires the qualified loader policy.') + if (len(argv) < 8 or argv[1:4] != ['-I', '-S', '-B'] or argv[5] != '--cpu-launch' + or not absolute(argv[4]) or not absolute(argv[6]) or not absolute(argv[7])): + raise ProcessError('Captured CPU launch vector differs from reserved bootstrap.') + inventory = {item['path']: item for item in files} + desc_file = inventory.get(argv[6]); bootstrap_file = inventory.get(argv[4]) + raw = encoded(descriptor) + if (not desc_file or desc_file['bytes'] != len(raw) or desc_file['sha256'] != digest(raw) + or not bootstrap_file or argv[7] not in inventory + or Path(argv[4]).name != 'recipe_process.py' or Path(argv[6]).name != 'snapshot.json'): + raise ProcessError('CPU bootstrap, descriptor and entry must be exact captured files.') if len(encoded(value)) > SPEC_LIMIT: raise ProcessError('Custody specification exceeds its bound.') return decode(encoded(value)) @@ -152,10 +392,13 @@ def verify_native_loader_policy(owner_uid): os.close(descriptor) -def verify_captures(spec): +def verify_captures(spec, *, checkpoint=None, include_cpu=True): + if 'cpu_snapshot' in spec and include_cpu: + verify_cpu_snapshot(spec['cpu_snapshot'], checkpoint=checkpoint) if 'native_loader_policy' in spec: verify_native_loader_policy(spec['trusted_install_owner_uid']) for item in spec['captured_files']+spec['runtime_files']: + if checkpoint is not None: checkpoint() raw = read_regular(item['path'], item['bytes']) if len(raw) != item['bytes'] or digest(raw) != item['sha256']: raise ProcessError('Captured input/source identity changed.', 'stale') @@ -263,7 +506,7 @@ def poll(self): raise ProcessError('Terminal receipt differs from custodian event.') self.terminal = True events.append(event) - if self.ready is None and not self.lost and time.monotonic()-self.started > READY_SECONDS: + if self.ready is None and not self.lost and time.monotonic()-self.started > (CPU_READY_SECONDS if 'cpu_snapshot' in self.spec else READY_SECONDS): self.close_control(wait=False) raise ProcessError('Custodian readiness deadline exceeded.', 'unavailable') self.process.poll() @@ -410,12 +653,32 @@ def _helper(control_fd,spec_fd,exe_fd,directory_fd): for name in hashes: logs[name]=os.open(name+'.log',os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600,dir_fd=directory_fd) selector=selectors.DefaultSelector();selector.register(channel,selectors.EVENT_READ,'control') leader=None;code=None;go=False;stopping=None;reason='not_started';drained=False - started=time.monotonic();deadline=started+READY_SECONDS + started=time.monotonic();deadline=started+(CPU_READY_SECONDS if 'cpu_snapshot' in spec else READY_SECONDS) def stop(why): nonlocal stopping,reason if stopping is None: stopping=time.monotonic();reason=why + class PreparationStopped(Exception): + pass + def preparation_checkpoint(): + try: + message = _receive(channel) + except BlockingIOError: + return + except (OSError, ProcessError): + message = None + if message is None: + selector.unregister(channel) + stop('controller_lost') + elif (type(message) is dict and set(message) == {'type', 'reason'} + and message['type'] == 'cancel' and message['reason'] in ('cancelled', 'shutdown', 'protocol_error')): + stop(message['reason']) + else: + stop('protocol_error') + raise PreparationStopped() try: - verify_captures(spec) + # Parent launch already verified the complete CPU snapshot. Keep READY + # responsive; recheck it cancellably at GO, before any recipe fork. + verify_captures(spec, include_cpu=False) _packet(channel,dict(common,type='ready',helper=who)) while True: now=time.monotonic() @@ -450,7 +713,10 @@ def stop(why): stop(message['reason']);continue expected={'type':'go','attempt_id':spec['attempt_id'],'request_sha256':spec['request_sha256'],'helper':who} if message!=expected or go or stopping is not None:stop('protocol_error');continue - verify_captures(spec) + try: + verify_captures(spec, checkpoint=preparation_checkpoint if 'cpu_snapshot' in spec else None) + except PreparationStopped: + continue go=True;deadline=time.monotonic()+spec['wall_seconds'] input_fd=sealed('tuldok-stdin',base64.b64decode(spec['stdin_base64'])) pipes={name:os.pipe() for name in hashes} @@ -511,7 +777,131 @@ def stop(why): channel.close() +def _cpu_descriptor_file(path): + if not absolute(path): raise ProcessError('CPU descriptor path must be absolute.') + raw = read_regular(path, CPU_MANIFEST_LIMIT) + value = validate_cpu_snapshot_descriptor(decode(raw)) + if raw != encoded(value): raise ProcessError('CPU descriptor must be canonical.') + return value + + +def _cpu_origins(manifest, descriptor, bootstrap): + import sys + root = Path(descriptor['root']) + permitted = {str(root / item['path']) for item in manifest['files']} + virtual_modules = {} + virtual = {'torch.ops': ('torch._ops', '_Ops', '_ops.py'), + 'torch.classes': ('torch._classes', '_Classes', '_classes.py')} + for name, module in list(sys.modules.items()): + location = getattr(module, '__file__', None) + if name in virtual: + defining_name, class_name, pseudo_file = virtual[name] + defining = sys.modules.get(defining_name) + if (defining is None or type(module) is not vars(defining).get(class_name) + or location != pseudo_file or type(module).__module__ != defining_name): + raise ProcessError('Captured virtual module identity differs: ' + name, 'unavailable') + location = vars(defining).get('__file__') + if location not in permitted: + raise ProcessError('Captured virtual module definition differs: ' + name, 'unavailable') + virtual_modules[name] = {'pseudo_file': pseudo_file, 'defining_module': defining_name, + 'defining_path': location, 'class': class_name} + if location is None: continue + path = str(Path(location).absolute()) + if path != bootstrap and path not in permitted: + raise ProcessError('Python module originates outside captured runtime: ' + name, 'unavailable') + expected = [str(root / manifest['launch'][key]) for key in ('stdlib', 'dynload', 'packages')] + if sys.path != expected: + raise ProcessError('Captured Python import path changed.', 'unavailable') + return virtual_modules + + +def _cpu_mappings(manifest, descriptor): + root = Path(descriptor['root']) + permitted = {str(root / item['path']) for item in manifest['files'] if item['kind'] == 'elf'} + observed = set() + for line in Path('/proc/self/maps').read_text().splitlines(): + columns = line.split(maxsplit=5) + if len(columns) < 6 or columns[5].startswith('['): continue + path = columns[5] + if path not in permitted: + raise ProcessError('Native mapping originates outside captured runtime: ' + path[:512], 'unavailable') + observed.add(path) + for key in ('python', 'loader'): + if str(root / manifest['launch'][key]) not in observed: + raise ProcessError('Captured Python/loader mapping is absent.', 'unavailable') + return sorted(observed) + + +def _cpu_launch_runtime(descriptor_path, entry, arguments): + descriptor = _cpu_descriptor_file(descriptor_path) + manifest = verify_cpu_snapshot(descriptor) + verify_native_loader_policy(os.stat('/usr').st_uid) + root = Path(descriptor['root']); launch = manifest['launch'] + bootstrap = str(Path(__file__).absolute()) + if not absolute(entry) or not absolute(bootstrap): + raise ProcessError('Captured CPU entry and bootstrap paths must be absolute.') + # Fresh profile-owned environment: C locale avoids host locale/gconv mmaps; + # Python UTF-8 mode retains deterministic Unicode filesystem/text behavior. + # No inherited loader, Python, site-hook or package-path overrides. + env = {'LANG': 'C', 'LC_ALL': 'C', 'PYTHONCOERCECLOCALE': '0', 'PYTHONUTF8': '1'} + env.update(PYTHONHOME=str(root / launch['home']), PYTHONNOUSERSITE='1', + PYTHONDONTWRITEBYTECODE='1', CUDA_VISIBLE_DEVICES='') + command = [str(root / launch['loader']), '--inhibit-cache', '--library-path', + ':'.join(str(root / item) for item in launch['library_dirs']), + '--glibc-hwcaps-mask', '', str(root / launch['python']), '-P', '-S', '-B', + bootstrap, '--cpu-child', descriptor_path, entry, *arguments] + os.execve(command[0], command, env) + + +def _cpu_child(descriptor_path, entry, arguments): + import sys + import runpy + descriptor = _cpu_descriptor_file(descriptor_path) + manifest = verify_cpu_snapshot(descriptor) + root = Path(descriptor['root']); launch = manifest['launch'] + if sys.version_info[:2] != (3, 12) or Path(sys.prefix) != root / launch['home']: + raise ProcessError('Captured Python bootstrap identity differs.', 'unavailable') + # Check already imported bootstrap modules before permitting package imports. + # -P -S excludes cwd, script directory, site hooks and user-site processing. + expected = [str(root / launch[key]) for key in ('stdlib', 'dynload', 'packages')] + initial = [str(root / launch['home'] / 'lib' / 'python312.zip'), *expected[:2]] + if sys.path != initial: + raise ProcessError('Captured Python initial path differs.', 'unavailable') + sys.path[:] = expected + bootstrap = str(Path(__file__).absolute()) + _cpu_origins(manifest, descriptor, bootstrap) + _cpu_mappings(manifest, descriptor) + import numpy + import torch + if str(numpy.__version__) != manifest['packages']['numpy'] or str(torch.__version__) != manifest['packages']['torch']: + raise ProcessError('Captured CPU package versions differ.', 'unavailable') + if torch.version.cuda is not None: + raise ProcessError('Captured profile must be CPU-only.', 'unsupported') + virtual_modules = _cpu_origins(manifest, descriptor, bootstrap) + mappings = _cpu_mappings(manifest, descriptor) + evidence = {'tuldok_cpu_runtime': {'snapshot_sha256': descriptor['manifest_sha256'], + 'python': sys.version, 'sys_path': list(sys.path), 'virtual_modules': virtual_modules, + 'environment': {key: os.environ.get(key) for key in ('LANG', 'LC_ALL', 'PYTHONCOERCECLOCALE', + 'PYTHONUTF8', 'PYTHONHOME', 'PYTHONNOUSERSITE', 'PYTHONDONTWRITEBYTECODE', 'CUDA_VISIBLE_DEVICES')}, + 'packages': {'numpy': str(numpy.__version__), 'torch': str(torch.__version__)}, + 'module_paths': {'numpy': numpy.__file__, 'torch': torch.__file__, + 'os': os.__file__, 'ctypes': ctypes.__file__}, 'mappings': mappings}} + line = encoded(evidence) + if len(line) > STREAM_LIMIT // 2: + raise ProcessError('Captured CPU initialization evidence exceeds its bound.', 'unsupported') + print(line.decode(), flush=True) + if not absolute(entry): raise ProcessError('Captured recipe entry must be absolute.') + sys.argv[:] = [entry, *arguments] + # Recipe source lookup starts only after the captured runtime qualification. + sys.path.insert(0, str(Path(entry).parent)) + runpy.run_path(entry, run_name='__main__') + + if __name__=='__main__': import sys - if len(sys.argv)!=6 or sys.argv[1]!='--helper':raise SystemExit(2) - _helper(*(int(value) for value in sys.argv[2:])) + if len(sys.argv)==6 and sys.argv[1]=='--helper': + _helper(*(int(value) for value in sys.argv[2:])) + elif len(sys.argv)>=4 and sys.argv[1] in ('--cpu-launch', '--cpu-child'): + (_cpu_launch_runtime if sys.argv[1]=='--cpu-launch' else _cpu_child)(sys.argv[2], sys.argv[3], sys.argv[4:]) + else: + raise SystemExit(2) diff --git a/runs.py b/runs.py index 8d59f1b0..34af5e61 100644 --- a/runs.py +++ b/runs.py @@ -469,7 +469,8 @@ def _execute_external(self, run_id): if recipe_contract.host_identity() != run['material']['host_files']: raise WorkbenchError('Recipe host changed after approval. Reopen and review a new attempt.', 'stale', 409) handle = recipe_process.launch(run['spec'], self._external_directory(run) / 'custody') - ready_deadline = time.monotonic() + 12 + preparation_seconds = recipe_process.CPU_READY_SECONDS if 'cpu_snapshot' in run['spec'] else recipe_process.READY_SECONDS + ready_deadline = time.monotonic() + preparation_seconds + 2 custody_deadline = ready_deadline cancel_deadline = None terminal = False @@ -492,7 +493,10 @@ def _execute_external(self, run_id): self._save(run) # No GO is sent when either transaction acknowledgement is unknown. handle.go(handle.expected_identity['helper']) - custody_deadline = time.monotonic() + run['spec']['wall_seconds'] + 15 + # CPU snapshots are reverified after GO before recipe + # wall time starts. Keep that bounded preparation separate. + recheck_seconds = preparation_seconds if 'cpu_snapshot' in run['spec'] else 0 + custody_deadline = time.monotonic() + recheck_seconds + run['spec']['wall_seconds'] + 15 elif event['type'] in ('terminal', 'lost'): terminal = True if terminal: diff --git a/runtime_snapshots.py b/runtime_snapshots.py new file mode 100644 index 00000000..09ce69ca --- /dev/null +++ b/runtime_snapshots.py @@ -0,0 +1,370 @@ +"""Explicit offline CPU-runtime capture; immutable filesystem content is canonical.""" +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import struct +import uuid + +from access import Access, AccessError, authorized +from workbench import WorkbenchError, encode + +PROFILE = 'captured_cpu_python_v1' +MAX_FILES = 40000 +MAX_FILE_BYTES = 512 * 1024 * 1024 +MAX_BYTES = 2 * 1024 * 1024 * 1024 +MAX_MANIFEST = 16 * 1024 * 1024 +SYSTEM_LIB = Path('/usr/lib/x86_64-linux-gnu') +DEFAULT_RESERVE = 6 * 1024 * 1024 * 1024 +NAME = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_.+-]{0,159}') +HASH = re.compile(r'[a-f0-9]{64}') + + +def fail(message, code='invalid'): + raise WorkbenchError('CPU runtime: ' + message, code, 409 if code in ('conflict','unavailable','stale') else 400) + + +def canonical(value): + from recipe_process import encoded + return encoded(value) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def configuration(value): + required = {'schema_version','python_root','python_version','site_packages','packages'} + if type(value) is not dict or set(value) - {'exclude_paths','stdlib_exclude_paths'} != required or type(value['schema_version']) is not int or value['schema_version'] != 1: + fail('Use the closed version-1 explicit source configuration.') + if type(value['python_version']) is not str or not re.fullmatch(r'3\.\d+', value['python_version']): + fail('Python version must name a specific Python 3 minor version.') + for name in ('python_root','site_packages'): + if type(value[name]) is not str or not Path(value[name]).is_absolute(): fail('Source roots must be explicit absolute directories.') + if not Path(value[name]).is_dir(): fail('Configured source directory is unavailable.','unavailable') + packages=value['packages'] + if type(packages) is not list or not 1 <= len(packages) <= 128 or any(type(x) is not str or not NAME.fullmatch(x) for x in packages) or len(set(packages)) != len(packages): + fail('Choose 1–128 unique explicit package entries.') + if not {'torch','numpy'} <= set(packages): fail('The CPU profile requires explicit Torch and NumPy packages.') + if any(name.endswith('.pth') for name in packages): fail('Automatic .pth hooks cannot be selected.') + excluded=value.get('exclude_paths',[]) + if type(excluded) is not list or len(excluded)>32 or any(type(item) is not str or item.startswith('/') or any(part in ('','.', '..') for part in item.split('/')) or item.split('/')[0] not in packages or '.dist-info' in item or 'license' in item.lower() for item in excluded): + fail('Exclusions must be bounded explicit paths inside selected non-metadata packages.') + excluded=value.get('stdlib_exclude_paths',[]) + if type(excluded) is not list or len(excluded)>32 or any(type(item) is not str or not item.startswith('lib-dynload/') or len(item.split('/'))!=2 or not NAME.fullmatch(item.split('/')[1]) or not item.endswith('.so') for item in excluded): + fail('Stdlib exclusions must name explicit optional lib-dynload extensions.') + return json.loads(encode(value)) + + +def file_identity(path, *, destination=None): + """Stream one no-follow regular source, optionally into a new private file.""" + fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK) + out=None + try: + before=os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_FILE_BYTES: fail('Source must be a bounded regular file.') + if destination is not None: out=os.open(destination,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + digest=hashlib.sha256(); count=0; prefix=b'' + while True: + raw=os.read(fd,1024*1024) + if not raw: break + if not prefix: prefix=raw[:4] + count+=len(raw) + if count>MAX_FILE_BYTES: fail('Source exceeds the per-file limit.') + digest.update(raw) + if out is not None: + view=memoryview(raw) + while view: view=view[os.write(out,view):] + after=os.fstat(fd) + if count!=before.st_size or (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns)!=(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns): + fail('Source changed during capture.','stale') + mode=0o500 if prefix==b'\x7fELF' else 0o400 + if out is not None: os.fchmod(out,mode);os.fsync(out) + return {'bytes':count,'sha256':digest.hexdigest(),'mode':mode,'kind':'elf' if prefix==b'\x7fELF' else 'data'} + finally: + os.close(fd) + if out is not None:os.close(out) + + +def elf_metadata(path): + """Read bounded ELF headers/dynamic strings only; never launch loader tools.""" + fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK) + try: + info=os.fstat(fd) + def read(offset,size): + if offset<0 or size<0 or offset+size>info.st_size or size>16*1024*1024: fail('Malformed ELF bounds.') + raw=os.pread(fd,size,offset) + if len(raw)!=size: fail('ELF changed while reading.','stale') + return raw + header=read(0,64) + if header[:7]!=b'\x7fELF\x02\x01\x01' or struct.unpack_from('info.st_size: fail('Invalid ELF segment.') + if kind==3: + if interpreter is not None or not 1<=length<=256:fail('Invalid ELF interpreter.') + raw=read(pos,length) + if raw[-1:]!=b'\0' or b'\0' in raw[:-1]:fail('Invalid ELF interpreter.') + interpreter=raw[:-1].decode('ascii') + if interpreter!='/lib64/ld-linux-x86-64.so.2': fail('Unsupported ELF interpreter.','unsupported') + if kind==2: + if seen or length%16 or length>65536:fail('Invalid ELF dynamic table.') + seen=True + for at in range(pos,pos+length,16): + tag,value=struct.unpack('=len(strings) or end<0:fail('Invalid ELF string offset.') + return strings[at:end].decode('ascii') + needed=[string(value) for tag,value in dynamic if tag==1] + if any(not NAME.fullmatch(name) for name in needed):fail('Native dependency must be a literal basename.') + search=[] + for tag,value in dynamic: + if tag in (15,29):search.extend(string(value).split(':')) + if any(not (entry=='$ORIGIN' or entry.startswith('$ORIGIN/')) or '$' in entry[len('$ORIGIN'):] for entry in search): + fail('Native search paths must be contained $ORIGIN paths.','unsupported') + after=os.fstat(fd) + if (info.st_size,info.st_mtime_ns,info.st_ctime_ns)!=(after.st_size,after.st_mtime_ns,after.st_ctime_ns):fail('ELF changed during inspection.','stale') + return needed,search,interpreter + finally:os.close(fd) + + +def relative_join(directory,suffix): + parts=[] + for part in (directory+'/'+suffix).split('/'): + if part in ('','.'):continue + if part=='..': + if not parts:fail('Native ORIGIN path escapes captured root.') + parts.pop() + else:parts.append(part) + return '/'.join(parts) + + +def inventory(config): + config=configuration(config) + python=Path(config['python_root']).resolve(strict=True);site=Path(config['site_packages']).resolve(strict=True) + version=config['python_version'];allowed=(python,site,SYSTEM_LIB.resolve()) + sources={};rows={};packages={};total_bytes=0 + def add(source,target): + nonlocal total_bytes + if target in rows:return + if len(rows)>=MAX_FILES:fail('Runtime file-count limit exceeded.') + resolved=source.resolve(strict=True) + if not any(resolved.is_relative_to(root) for root in allowed):fail('Source symlink escapes configured runtime roots.') + if not resolved.is_file():fail('Special source file is unsupported.') + rows[target]=dict(path=target,**file_identity(resolved));sources[target]=resolved + total_bytes+=rows[target]['bytes'] + if total_bytes>MAX_BYTES:fail('Runtime exceeds total-byte limit.') + def tree(source,prefix,exclude=()): + excluded={'packages/'+item for item in config.get('exclude_paths',[])} | {'python/lib/python'+version+'/'+item for item in config.get('stdlib_exclude_paths',[])} + def omitted(path): + target=prefix+'/'+path.relative_to(source).as_posix() + return any(target==item or target.startswith(item+'/') for item in excluded) + for directory,dirs,files in os.walk(source,followlinks=False): + dirs[:]=sorted(name for name in dirs if not (Path(directory)==source and name in exclude) and not omitted(Path(directory)/name)) + if any((Path(directory)/name).is_symlink() for name in dirs):fail('Directory symlinks are unsupported.') + for name in sorted(files): + item=Path(directory)/name + if omitted(item):continue + add(item,prefix+'/'+item.relative_to(source).as_posix()) + add(python/'bin'/('python'+version),'python/bin/python'+version) + tree(python/'lib'/('python'+version),'python/lib/python'+version,('site-packages',)) + for path in sorted((python/'lib').glob('libpython'+version+'.so*')):add(path,'python/lib/'+path.name) + for name in config['packages']: + entry=site/name + if entry.is_symlink():fail('Selected package roots cannot be symlinks.') + if entry.is_dir():tree(entry,'packages/'+name) + else:add(entry,'packages/'+name) + if name.endswith('.dist-info'): + raw=(entry/'METADATA').read_bytes() + if len(raw)>1024*1024:fail('Package metadata exceeds bound.') + values={} + for line in raw.decode('utf-8').splitlines(): + if line.startswith(('Name: ','Version: ')): + key,value=line.split(': ',1);values[key]=value + if set(values)!={'Name','Version'} or values['Name'] in packages:fail('Package identity metadata is invalid.') + packages[values['Name'].lower().replace('_','-')]=values['Version'] + if not {'torch','numpy'}<=set(packages):fail('Include exact Torch and NumPy distribution metadata.') + add(SYSTEM_LIB/'ld-linux-x86-64.so.2','system/lib/ld-linux-x86-64.so.2') + processed=set() + while True: + pending=[path for path,row in rows.items() if row['kind']=='elf' and path not in processed] + if not pending:break + for target in pending: + try:needed,search,_interpreter=elf_metadata(sources[target]) + except WorkbenchError as error:fail('Unqualified ELF '+target+': '+str(error),error.code) + processed.add(target) + origin=str(PurePosixPath(target).parent) + directories=[relative_join(origin,item[len('$ORIGIN'):]) for item in search] + native_dirs=sorted({str(PurePosixPath(path).parent) for path,row in rows.items() if row['kind']=='elf'}) + for name in needed: + matches=[directory+'/'+name for directory in [*directories,'python/lib',*native_dirs,'system/lib'] if directory+'/'+name in rows] + if matches: + if len({rows[item]['sha256'] for item in matches})!=1:fail('Ambiguous captured native dependency: '+name) + continue + source=python/'lib'/name + if source.exists():add(source,'python/lib/'+name) + elif (SYSTEM_LIB/name).exists():add(SYSTEM_LIB/name,'system/lib/'+name) + else:fail('Native dependency is missing: '+name,'unavailable') + launch={'python':'python/bin/python'+version,'loader':'system/lib/ld-linux-x86-64.so.2', + 'home':'python','stdlib':'python/lib/python'+version,'dynload':'python/lib/python'+version+'/lib-dynload', + 'packages':'packages','library_dirs':sorted({str(PurePosixPath(path).parent) for path,row in rows.items() if row['kind']=='elf'}|{'python/lib','system/lib'})} + manifest={'schema_version':1,'profile':PROFILE,'python_version':version,'packages':dict(sorted(packages.items())), + 'launch':launch,'files':[rows[path] for path in sorted(rows)],'total_bytes':sum(row['bytes'] for row in rows.values())} + from recipe_process import validate_cpu_manifest + manifest=validate_cpu_manifest(manifest) + raw=canonical(manifest) + if len(raw)>MAX_MANIFEST:fail('Runtime manifest exceeds limit.') + return manifest,sources + + +def descriptor(manifest,root,catalog_id): + raw=canonical(manifest) + return {'schema_version':1,'profile':PROFILE,'catalog_id':catalog_id,'root':str(root), + 'manifest_sha256':sha(raw),'manifest_bytes':len(raw),'file_count':len(manifest['files']), + 'total_bytes':manifest['total_bytes'],'python_version':manifest['python_version'], + 'packages':manifest['packages'],'launch':manifest['launch']} + + +def sync_directory(path): + fd=os.open(path,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + try:os.fsync(fd) + finally:os.close(fd) + + +def publish_directory(source,destination): + libc=ctypes.CDLL(None,use_errno=True) + function=getattr(libc,'renameat2',None) + if function is None:fail('No-replace directory publication is unavailable.','unsupported') + if function(-100,os.fsencode(source),-100,os.fsencode(destination),1): + code=ctypes.get_errno() + raise OSError(code,os.strerror(code)) + + +class RuntimeSnapshots: + def __init__(self,db,access,catalog_path): + self.db,self.access=db,access + self.path=Path(catalog_path).resolve()/'runtime-snapshots' + + @authorized('admin','runtime_snapshots.inspect',owner_only=True) + def inspect(self,config): + manifest,_sources=inventory(config);raw=canonical(manifest) + return {'inventory_sha256':sha(raw),'manifest':manifest,'file_count':len(manifest['files']), + 'total_bytes':manifest['total_bytes'],'manifest_bytes':len(raw),'free_bytes':shutil.disk_usage(self.db.path).free} + + @authorized('inspect','runtime_snapshots.get') + def get(self,manifest_sha256,verify=True): + if type(manifest_sha256) is not str or not HASH.fullmatch(manifest_sha256):fail('Invalid snapshot SHA256.') + from recipe_process import read_regular,decode,validate_cpu_manifest,verify_cpu_snapshot,ProcessError + root=self.path/manifest_sha256 + try: + raw=read_regular(root/'manifest.json',MAX_MANIFEST) + if sha(raw)!=manifest_sha256:fail('Snapshot manifest hash differs.','stale') + manifest=validate_cpu_manifest(decode(raw)) + if canonical(manifest)!=raw:fail('Snapshot manifest encoding differs.','stale') + value=descriptor(manifest,root,self.db.dataset_id) + if verify:verify_cpu_snapshot(value) + return value + except FileNotFoundError: + fail('Snapshot is unavailable.','unavailable') + except (OSError,ProcessError) as error: + fail('Snapshot is unavailable or differs from its retained identity: '+str(error),'stale') + + @authorized('admin','runtime_snapshots.create',owner_only=True) + def create(self,config,expected_inventory_sha256,*,quiesced=False,reserve_bytes=DEFAULT_RESERVE): + if quiesced is not True:fail('Snapshot creation requires explicit --quiesced.','conflict') + if type(reserve_bytes) is not int or reserve_bytes<0:fail('Reserve bytes must be a nonnegative integer.') + if not self.db.claim_runtime('workspace_runs'):fail('Stop the active application before creating a runtime snapshot.','conflict') + with self.db.lock: + for row in self.db.execute('SELECT document FROM workspace_runs'): + value=json.loads(row[0]) + if value.get('state') in ('queued','running','cancelling','unknown'):fail('Resolve retained active or unknown Runs first.','conflict') + self.access.require('admin','runtime_snapshots.create',owner_only=True) + manifest,sources=inventory(config);raw=canonical(manifest);identity=sha(raw) + if identity!=expected_inventory_sha256:fail('Runtime inventory changed after inspection.','stale') + self.path.mkdir(mode=0o700,exist_ok=True) + if self.path.is_symlink():fail('Snapshot directory cannot be a symlink.') + target=self.path/identity + if target.exists():return self.get(identity) + if shutil.disk_usage(self.path).freenode.type==='execute'),input=graph.nodes.find(node=>node.type==='input').parameters; - root.innerHTML=`

Saved recipes

External recipe

${isDirty()?'Unsaved changes':'Saved r'+graph.revision}
1 · Project inputs2 · Execute captured recipe3 · Verify outputs
${fieldsHTML(execute.parameters,definitions,'external-param-')}
Exact project and input references
${esc(JSON.stringify(input,null,2))}

Execution approval

${permissions}

${preflight?`

Material SHA256 ${esc(preflight.material_sha256)}

Graph r${esc(preflight.graph_revision)} · runtime ${esc(preflight.runtime?.profile||preflight.runtime?.id||'declared runtime')}

Captured sources, inputs, runtime and limits
${esc(JSON.stringify(preflight.material,null,2))}
Declared output roles
${esc(JSON.stringify(preflight.outputs,null,2))}
`:'

Preflight checks identity without executing project code.

'}
${owner()?`
Grant this material to an actor${grantReceipt?`

Approval ${esc(grantReceipt.id||grantReceipt.approval_id)} · actor ${esc(grantReceipt.consumer_actor_id||recipient)}

`:''}
`:`

The grant must bind your actor, this graph revision and this exact material digest.

`}${pending?`

Request ${esc((pending.body||pending).request_key)} for graph ${esc((pending.body||pending).graph_id)} r${esc((pending.body||pending).graph_revision)} has an unresolved acknowledgement. Originating actor ${esc(pending.actor_id||'unknown')}. Reconcile this same request before starting another.

`:''}

Execution receipts establish process and artifact custody, not scientific correctness.

`; + root.innerHTML=`

Saved recipes

External recipe

${isDirty()?'Unsaved changes':'Saved r'+graph.revision}
1 · Project inputs2 · Execute captured recipe3 · Verify outputs
${fieldsHTML(execute.parameters,definitions,'external-param-')}
Exact project and input references
${esc(JSON.stringify(input,null,2))}

Execution approval

${permissions}

${preflight?`

Material SHA256 ${esc(preflight.material_sha256)}

Graph r${esc(preflight.graph_revision)} · runtime ${esc(preflight.material?.recipe?.runtime||preflight.runtime?.profile||preflight.runtime?.id||'declared runtime')}

Captured sources, inputs, runtime and limits
${esc(JSON.stringify(preflight.material,null,2))}
Declared output roles
${esc(JSON.stringify(preflight.outputs,null,2))}
`:'

Preflight checks identity without executing project code.

'}
${owner()?`
Grant this material to an actor${grantReceipt?`

Approval ${esc(grantReceipt.id||grantReceipt.approval_id)} · actor ${esc(grantReceipt.consumer_actor_id||recipient)}

`:''}
`:`

The grant must bind your actor, this graph revision and this exact material digest.

`}${pending?`

Request ${esc((pending.body||pending).request_key)} for graph ${esc((pending.body||pending).graph_id)} r${esc((pending.body||pending).graph_revision)} has an unresolved acknowledgement. Originating actor ${esc(pending.actor_id||'unknown')}. Reconcile this same request before starting another.

`:''}

Execution receipts establish process and artifact custody, not scientific correctness.

`; const $=id=>root.querySelector('#'+id); $('real-graph-select').onchange=event=>choose(event.target.value);$('real-refresh').onclick=refresh; $('external-name').oninput=event=>changeGraph(()=>graph.name=event.target.value,false); diff --git a/tests/browser_workspace_cpu.cjs b/tests/browser_workspace_cpu.cjs new file mode 100644 index 00000000..bf3495c1 --- /dev/null +++ b/tests/browser_workspace_cpu.cjs @@ -0,0 +1,106 @@ +'use strict'; +// Actual frozen-checkpoint CPU comparison; requires a qualified, caller-owned catalog/runtime snapshot. +const fixtureAuth=require('./auth_helpers.cjs').authenticatedFixture(),fetch=fixtureAuth.fetch; +const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); +const {spawn,execFileSync}=require('node:child_process'); +const {pageLoadTracker}=require('./browser_page_load.cjs'); +const {waitForDebugger}=require('./browser_startup.cjs'); +const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); +const root=path.resolve(__dirname,'..'),report=qaDirectory(root,'workspace-cpu-comparison'); +function option(name,environment){const index=process.argv.indexOf(name);return index<0?process.env[environment]:process.argv[index+1];} +const catalog=option('--catalog','TULDOK_CPU_CATALOG'),projectRoot=option('--project','TULDOK_CPU_PROJECT'),snapshot=option('--snapshot','TULDOK_CPU_SNAPSHOT'); +assert.ok(catalog&&projectRoot&&/^[a-f0-9]{64}$/.test(snapshot||''),'Explicit precreated --catalog, --project and --snapshot required'); +assert.ok(process.argv.includes('--runtime-qualified'),'Pass --runtime-qualified only after the captured runtime qualification passes'); +const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-cpu-browser-')); +const project=JSON.parse(fs.readFileSync(path.join(projectRoot,'project.json'))); +const sha=raw=>crypto.createHash('sha256').update(raw).digest('hex'); +const originalFiles=Object.fromEntries(fs.readdirSync(projectRoot).filter(name=>fs.statSync(path.join(projectRoot,name)).isFile()).map(name=>[name,sha(fs.readFileSync(path.join(projectRoot,name)))])); +const inspections=[],captures=[],checks=[],requests=[],errors=[],children=[],tracker=pageLoadTracker();const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms));let ws,send,evaluate,serverErrors=''; +async function until(fn,message='Timed out',seconds=60){for(let i=0;i['-u','app.py','--port',String(port),'--data',catalog,'--project-root','sample='+projectRoot]; +(async()=>{ + let server=launch('python3',serverArgs(0),{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); + const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); + const base='http://127.0.0.1:'+port; + await fixtureAuth.register('http://127.0.0.1:'+port,catalog,root); + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); + const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); + ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next,timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP timeout '+method));},20000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});ws.send(JSON.stringify({id,method,params}));}); + ws.onmessage=event=>{const message=JSON.parse(event.data);tracker.observe(message);if(message.id){const task=pending.get(message.id);pending.delete(message.id);if(task)message.error?task.reject(message.error):task.resolve(message.result);}if(message.method==='Runtime.exceptionThrown')errors.push(message.params.exceptionDetails);if(message.method==='Network.requestWillBeSent')requests.push({url:message.params.request.url,method:message.params.request.method});}; + evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + const doc=frame=>frame?'document.getElementById("catalog-frame").contentDocument':'document'; + const element=(selector,frame=false)=>`${doc(frame)}.querySelector(${JSON.stringify(selector)})`,text=(selector,frame=false)=>evaluate(`${element(selector,frame)}?.textContent`); + async function click(selector,frame=false){ + const box=await until(()=>evaluate(`(async()=>{const e=${element(selector,frame)};if(!e||e.disabled||!e.getClientRects().length)return null;${frame?'document.getElementById("catalog-frame").scrollIntoView({block:"start"});':''}e.scrollIntoView({block:'center'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));const b=e.getBoundingClientRect(),x=b.x+b.width/2,y=b.y+b.height/2,hit=${doc(frame)}.elementFromPoint(x,y),f=${frame?'document.getElementById("catalog-frame").getBoundingClientRect()':'{x:0,y:0}'};return hit&&(hit===e||e.contains(hit))?{x:x+f.x,y:y+f.y}:null})()`),'Click target unavailable '+selector); + for(const type of ['mousePressed','mouseReleased'])await send('Input.dispatchMouseEvent',{type,button:'left',clickCount:1,...box}); + } + async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Escape:27,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} + async function fill(selector,value,frame=false){await click(selector,frame);await key('a','KeyA',2);await send('Input.insertText',{text:value});} + const select=(selector,value,frame=false)=>evaluate(`(()=>{const e=${element(selector,frame)};e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); + async function nav(view){await evaluate(`document.getElementById('nav-${view}').focus()`);await key('Enter');await until(()=>evaluate(`!document.getElementById('view-${view}').hidden && !document.getElementById('project-${view}-root').hidden`));await until(()=>text(`#project-${view}-root`).then(value=>value?.includes('PROJECT · INSPECTION ONLY')));} + const check=value=>{checks.push(value);console.log('PASS',value);}; + async function reload(){const previous=(await send('Page.getFrameTree')).frameTree.frame;await send('Page.reload');await until(()=>tracker.reloaded(previous),'Main document reload not observed');} + async function openProject(manifest,name,id){await until(()=>evaluate('typeof document.getElementById("open-project")?.onclick==="function"'));await click('#open-project');await until(()=>evaluate('document.getElementById("project-open-dialog").open && !document.getElementById("project-open-submit").disabled'));await select('#project-root','sample');await fill('#project-manifest',manifest);await evaluate('document.getElementById("project-open-submit").focus()');await key('Enter');await until(()=>evaluate('document.getElementById("dataset").value==='+JSON.stringify('project:'+id)));await nav('data');assert.match(await text('#project-data-root'),new RegExp(name.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')));} + await send('Page.enable');await send('Page.setLifecycleEventsEnabled',{enabled:true});await send('Runtime.enable');await send('Network.enable'); + const offlineCanary=process.env.LD_PRELOAD?JSON.parse(execFileSync('python3',['-c',"import sys;sys.path.insert(0,'tests');from test_workspace_offline import CANARY;exec(CANARY)"],{cwd:root,encoding:'utf8'})):null; + await fixtureAuth.browser(send,'http://127.0.0.1:'+port); + await send('Page.navigate',{url:base+'/workspaces'});await until(()=>evaluate('!!document.getElementById("open-project")')); + async function captureWorkspace(name,width,selector){await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});await evaluate(`${element(selector)}.scrollIntoView({block:'start'})`);await pause(100);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'));const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Actual frozen-checkpoint CPU comparison; exposed seeds, no retraining/GPU acceptance'});} + async function api(route,body){const response=await fetch(base+route,body===undefined?{}:{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});const value=await response.json();assert.ok(response.ok,JSON.stringify({route,status:response.status,value}));return value;} + async function artifact(run,name){const item=run.artifacts.find(item=>item.name===name);assert.ok(item,name);const response=await fetch(base+item.url);assert.equal(response.status,200);const bytes=Buffer.from(await response.arrayBuffer());assert.equal(sha(bytes),item.sha256);return bytes;} + async function completed(id){let logged=0;return until(async()=>{const run=await api('/api/workspace/runs/'+id);assert.ok(!['failed','unknown','cancelled','interrupted'].includes(run.state),JSON.stringify({state:run.state,error:run.error,events:run.events}));if(Date.now()-logged>30000){console.log('CPU run',run.id,run.state,run.custody_phase);logged=Date.now();}return run.state==='completed'?run:null;},'Actual CPU comparison did not complete',750);} + await openProject('project.json',project.name,project.id);await nav('pipeline'); + await click('[data-project-recipe="compare"]');await until(()=>evaluate('!!document.querySelector("[data-external-input=checkpoint]")')); + for(const input of ['checkpoint','freeze','build_receipt'])await select('[data-external-input="'+input+'"]',input); + await fill('#external-create-name','CPU comparison · exposed seeds 900–903'); + await evaluate('document.getElementById("external-create-submit").focus()');await key('Enter');await until(()=>evaluate('!!document.getElementById("external-preflight")')); + await fill('#external-param-threads','8');await fill('#external-param-repetitions','1');await click('#external-save');await until(()=>text('#external-dirty').then(value=>value?.startsWith('Saved'))); + await click('#external-preflight');await until(()=>evaluate('!document.getElementById("external-reviewed").disabled'), 'CPU preflight did not become approvable',120); + assert.match(await text('#external-material'),/Material SHA256/); + await captureWorkspace('cpu-pipeline-1440',1440,'#external-material');await captureWorkspace('cpu-pipeline-390',390,'#external-material'); + const previousRunIds=new Set((await api('/api/workspace/runs')).map(run=>run.id)); + await click('#external-reviewed');await evaluate('document.getElementById("external-submit").focus()');await key('Enter'); + const submitted=await until(async()=>{const runs=await api('/api/workspace/runs');return runs.find(run=>run.execution_kind==='external_recipe'&&!previousRunIds.has(run.id));},'CPU Run was not submitted'); + fs.writeFileSync(path.join(report,'submitted-run.json'),JSON.stringify(submitted,null,2)); + const run=await completed(submitted.id); + fs.writeFileSync(path.join(report,'completed-run.json'),JSON.stringify(run,null,2)); + assert.equal(run.execution_kind,'external_recipe');assert.equal(run.material.parameters.threads,8);assert.equal(run.material.parameters.repetitions,1); + assert.ok(JSON.stringify(run.material).includes(snapshot),'Run material does not bind caller-qualified snapshot'); + await until(()=>text('#real-run-state').then(value=>value==='completed')); + await captureWorkspace('cpu-runs-390',390,'.external-run-custody');await captureWorkspace('cpu-runs-1440',1440,'.external-run-custody'); + check('Actual approved CPU graph completes all original methods, frames and exposed seeds'); + const artifactDirectory=path.join(report,'artifacts');fs.mkdirSync(artifactDirectory); + const names=['comparison-freeze.json','timing.json','result.json','input-model.pt','input-freeze.json','comparison-receipt.json']; + const downloaded={};for(const name of names){downloaded[name]=await artifact(run,name);fs.writeFileSync(path.join(artifactDirectory,name),downloaded[name]);} + // Explicit owner-local QA observation, not an HTTP artifact or agent API acceptance claim. + const localEvidence=path.join(report,'owner-local-custody');fs.mkdirSync(localEvidence); + const published=path.join(catalog,'workspace-runs',run.id),custody=path.join(catalog,'workspace-runs','.external-'+run.id,'custody'); + const readLocal=(directory,name)=>{const raw=fs.readFileSync(path.join(directory,name));fs.writeFileSync(path.join(localEvidence,name),raw);return raw;}; + const manifest=JSON.parse(readLocal(published,'artifact-manifest.json')); + assert.equal(manifest.run_id,run.id);assert.equal(manifest.material_sha256,run.material_sha256); + assert.deepEqual(manifest.artifacts,run.artifacts.map(({url,locked,...item})=>item)); + const custodyRaw=readLocal(published,'custody-receipt.json'),custodyReceipt=JSON.parse(custodyRaw); + assert.equal(sha(custodyRaw),manifest.custody_receipt_sha256);assert.equal(custodyReceipt.attempt_id,run.id);assert.equal(custodyReceipt.request_sha256,manifest.request_sha256); + assert.equal(custodyReceipt.outcome,'completed');for(const flag of ['go_received','spawn_closed','drained','echild'])assert.equal(custodyReceipt[flag],true); + const liveReceipt=fs.readFileSync(path.join(custody,'receipt.json'));assert.deepEqual(JSON.parse(liveReceipt),custodyReceipt); + const streams={};for(const name of ['stdout','stderr']){streams[name]=readLocal(custody,name+'.log');assert.equal(streams[name].length,custodyReceipt.streams[name].bytes);assert.equal(sha(streams[name]),custodyReceipt.streams[name].sha256);} + const runtimeLines=streams.stdout.toString('utf8').split('\n').filter(line=>line.startsWith('{')&&line.includes('"tuldok_cpu_runtime"')).map(line=>JSON.parse(line)); + assert.equal(runtimeLines.length,1);const runtimeProof=runtimeLines[0].tuldok_cpu_runtime;assert.equal(runtimeProof.snapshot_sha256,snapshot);assert.ok(runtimeProof.packages.numpy&&runtimeProof.packages.torch&&runtimeProof.mappings&&runtimeProof.module_paths); + fs.writeFileSync(path.join(localEvidence,'runtime-proof.json'),JSON.stringify({source:'Explicit owner-local QA read of hash-bound custodian stdout; not HTTP download or agent API parity',...runtimeLines[0]},null,2)); + check('Owner-local QA custody logs retain exact hash-bound CPU bootstrap module and mapping evidence'); + const oracle=JSON.parse(execFileSync('python3',['tests/qualify_cpu_comparison.py','--artifacts',artifactDirectory,'--project',projectRoot,'--report',path.join(report,'cold-report.json')],{cwd:root,encoding:'utf8'})); + assert.equal(oracle.timed_frames,1440);assert.equal(oracle.native_launches,24); + check('Independent cold audit verifies original checkpoint/source bytes, complete reports, medians and strict accuracy thresholds'); + server.kill('SIGINT');await until(()=>server.exitCode!==null);server=launch('python3',serverArgs(port),{cwd:root,stdio:['ignore','pipe','pipe']});let reopened='';server.stdout.on('data',chunk=>reopened+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk);await until(()=>reopened.includes('Tuldok:'));await reload(); + const retained=await api('/api/workspace/runs/'+run.id);fs.writeFileSync(path.join(report,'reopened-run.json'),JSON.stringify(retained,null,2));assert.equal(retained.state,'completed');assert.equal(retained.material_sha256,run.material_sha256); + for(const name of names)assert.deepEqual(await artifact(retained,name),downloaded[name]); + for(const [name,hash] of Object.entries(originalFiles))assert.equal(sha(fs.readFileSync(path.join(projectRoot,name))),hash); + assert.deepEqual(errors,[]);assert.ok(!serverErrors.includes('Traceback'),serverErrors); + check('Server restart preserves exact completed comparison artifacts and caller-owned project inputs'); + fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({source_commit:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),dirty:!!execFileSync('git',['status','--porcelain'],{cwd:root,encoding:'utf8'}).trim(),qualification:'Fresh frozen-checkpoint CPU correctness/backend comparison through actual approved Run',previously_exposed_seeds:[900,901,902,903],fresh_training_acceptance:false,gpu_acceptance:false,full_accuracy_fields_retained:false,checks,captures,run_id:run.id,material_sha256:run.material_sha256,runtime_snapshot_sha256:snapshot,local_qa_evidence:{directory:'owner-local-custody',custody_receipt_sha256:sha(custodyRaw),http_download_acceptance:false},cold:oracle,offline_canary:offlineCanary,browser:(await send('Browser.getVersion')).product},null,2));console.log('PASS CPU comparison browser:',report); +})().catch(async error=>{console.error(error);console.error('Application stderr:',serverErrors);if(evaluate)try{console.error(await evaluate('JSON.stringify({url:location.href,body:document.body.innerText.slice(-6000)})'));}catch{}process.exitCode=1;}).finally(()=>{ws?.close();for(const child of children.reverse())child.kill('SIGTERM');}); diff --git a/tests/qualify_captured_cpu_runtime.py b/tests/qualify_captured_cpu_runtime.py new file mode 100644 index 00000000..14139900 --- /dev/null +++ b/tests/qualify_captured_cpu_runtime.py @@ -0,0 +1,224 @@ +"""Actual captured CPU qualification through normal approved production Runs. + +No mocks, package installation, runtime copying, historical model loading or +scientific/model acceptance. Requires an existing owner and published snapshot. +""" +import argparse +import ast +import hashlib +import json +from pathlib import Path +import struct +import sys +import time +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app import Dataset +from recipe_process import validate_receipt + +PROBE = r'''import hashlib,json,sys,time +from pathlib import Path +import numpy as np +import torch +mode, destination = sys.argv[1:] +out = Path(destination) +torch.set_num_threads(2) +if mode == 'sleep': + print('cpu_probe_sleep_ready', flush=True) + time.sleep(120) + raise RuntimeError('Cancellation qualification was not requested in time') +assert mode == 'compute' +x = torch.arange(12, dtype=torch.float64).reshape(3,4).requires_grad_() +w = torch.arange(8, dtype=torch.float64).reshape(4,2).requires_grad_() +y = x @ w +loss = (y*y).sum() +loss.backward() +expected_y = [[sum(float(i*4+k)*float(k*2+j) for k in range(4)) for j in range(2)] for i in range(3)] +expected_dx = [[sum(2*expected_y[i][j]*float(k*2+j) for j in range(2)) for k in range(4)] for i in range(3)] +expected_dw = [[sum(float(i*4+k)*2*expected_y[i][j] for i in range(3)) for j in range(2)] for k in range(4)] +assert y.tolist() == expected_y and x.grad.tolist() == expected_dx and w.grad.tolist() == expected_dw +assert all(value.device.type == 'cpu' for value in (x,w,y,x.grad,w.grad)) +array = np.arange(12, dtype=' 0 if positive else value >= 0) and value <= maximum, + 'Invalid/nonfinite/out-of-bound numerical report') + return value + + +def audit(directory, threads=8, repetitions=1, project=None): + directory = Path(directory) + raw = {name: (directory/name).read_bytes() for name in NAMES} + hashes = {name: hashlib.sha256(value).hexdigest() for name, value in raw.items()} + decode = lambda name: json.loads(raw[name], parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value))) + config, timing, result, receipt = (decode(name) for name in NAMES[:3]+('comparison-receipt.json',)) + require(hashes['input-model.pt'] == CHECKPOINT and hashes['input-freeze.json'] == FREEZE, + 'Original checkpoint/freeze bytes changed') + require(receipt['schema_version'] == 1 and receipt['comparison'] == 'fresh_cpu_checkpoint_comparison', 'Wrong receipt') + require(receipt['source_sha256'] == SOURCES, 'Original algorithm sources changed') + require(receipt['outputs'] == {name: hashes[name] for name in NAMES[:-1]}, 'Output custody mismatch') + require(config == timing['config'] == result['config'], 'Configuration drift') + require(result['status'] == 'verified', 'Original comparison not verified') + expected = {'schema': 'rheon-cpu-only-comparison-v1', 'device': 'cpu', 'cuda_visible_devices': '', + 'all_model_tensors_on_cpu': True, 'n': 32, 'frames': 120, 'seeds': [900, 901, 902, 903], + 'native_threads_all_methods': threads, 'torch_threads': threads, 'torch_interop_threads': 1, + 'repetitions': repetitions, 'checkpoint_sha256': CHECKPOINT, 'binary_sha256': BINARY} + for key, value in expected.items(): + require(config[key] == value, 'Configuration mismatch: '+key) + require('previously exposed seeds' in config['test_status'] and 'no retraining' in config['test_status'], 'Missing exposure disclosure') + require(config['source_sha256'] == {key: value for key, value in SOURCES.items() if key != 'verify_parallel_results.py'}, 'Configuration source drift') + for key in ('model_setup_s', 'warmup_s'): + number(config[key]) + for key, value in timing.items(): + if key != 'python_peak_rss_bytes': + require(result[key] == value, 'Timing/final result drift: '+key) + require(number(result['python_peak_rss_bytes']) >= number(timing['python_peak_rss_bytes']), 'Peak RSS decreased') + methods = ('jacobi', 'sgs', 'learned') + identities = [(repeat, seed, method) for repeat in range(repetitions) for seed in range(900, 904) + for method in methods[(repeat+seed-900) % 3:]+methods[:(repeat+seed-900) % 3]] + records = result['records'] + require([(r['repetition'], r['seed'], r['method']) for r in records] == identities, 'Missing, duplicated, or reordered timed trajectory') + residuals, divergences, courants = [], [], [] + for record in records: + require(record['frames'] == 120 and len(record['reports']) == 120, 'Missing timed frames') + number(record['wall_ms_including_native_startup_teardown'], positive=True) + for index, frame in enumerate(record['reports'], 1): + require(frame['frame'] == index and math.isclose(number(frame['time']), index/60, rel_tol=1e-12, abs_tol=1e-12), 'Wrong native frame/time order') + residuals.append(number(frame['relative_residual'], 1e-8)) + divergences.append(number(frame['actual_divergence_max'], 1e-5)) + courants.append(number(frame['courant'], 1)) + if record['method'] == 'learned': + require(frame['iterations'] == 0, 'Learned path used iterative corrections') + medians = {method: statistics.median(r['wall_ms_including_native_startup_teardown'] for r in records if r['method'] == method) for method in methods} + require(result['median_120_frame_wall_ms'] == medians, 'Incorrect medians') + for method in ('jacobi', 'sgs'): + require(result['speedup_vs_'+method] == medians[method]/medians['learned'], 'Incorrect timing ratio') + require(result['qualified_timed_frames'] == 1440*repetitions, 'Incorrect timed frame total') + require(result['max_relative_residual'] == max(residuals) and result['max_divergence'] == max(divergences), 'Incorrect report maxima') + accuracy = result['accuracy'] + require([case['seed'] for case in accuracy] == list(range(900, 904)), 'Missing accuracy seed') + accuracy_maxima = {} + for case in accuracy: + for method in ('jacobi', 'sgs'): + values = case[method] + require(len(values['frames']) == 120, 'Missing accuracy frames') + for field in ('velocity', 'tracer'): + maximum = max(number(frame[field+'_relative_l2']) for frame in values['frames']) + require(maximum < 1e-5 and values['max_'+field+'_relative_l2'] == maximum, 'Incorrect or failing accuracy maximum') + accuracy_maxima[f"{case['seed']}:{method}:{field}"] = maximum + launches = receipt['native_launches'] + require(Counter((item['seed'], item['method']) for item in launches) + == Counter({(seed, method): repetitions+1 for seed in range(900, 904) for method in methods}), 'Missing native launch custody') + policy = receipt['native_policy'] + expected_maps = {'sealed-native', 'sealed-loader'} | {entry['path'] for entry in policy['runtime_files'] if entry['path'] != policy['loader']['path']} + require(receipt['build_receipt']['binary']['sha256'] == BINARY, 'Native build identity mismatch') + for item in launches: + ready = item['ready'] + require(ready['n'] == 32 and ready['seed'] == item['seed'] and ready['method'] == item['method'], 'READY identity mismatch') + require(set(item['mappings']) == expected_maps, 'Native loaded mapping set differs from approved policy') + require(all(type(mapping['inode']) is int and mapping['inode'] > 0 for mapping in item['mappings'].values()), 'Invalid loaded mapping identity') + if project: + project = Path(project) + require((project/'model.pt').read_bytes() == raw['input-model.pt'] and (project/'freeze.json').read_bytes() == raw['input-freeze.json'], 'Project input bytes changed') + for name, digest in SOURCES.items(): + require(hashlib.sha256((project/'lab/scripts'/name).read_bytes()).hexdigest() == digest, 'Project source bytes changed') + return {'qualification': 'Independent retained-report arithmetic and exact input/output custody audit of actual CPU comparison', + 'limitations': ['Previously exposed seeds 900–903; no retraining or GPU acceptance.', + 'Full velocity/tracer norms were computed by the original producer in memory; this cold audit recomputes report maxima, not absent field arrays.', + 'Native integrity launch overhead is included; no historical timing reproduction claim.'], + 'artifact_sha256': hashes, 'torch': config['torch'], 'threads': threads, 'repetitions': repetitions, + 'timed_trajectories': len(records), 'timed_frames': len(residuals), 'accuracy_replays': len(accuracy), + 'native_launches': len(launches), 'median_120_frame_wall_ms': medians, + 'max_relative_residual': max(residuals), 'max_divergence': max(divergences), 'max_courant': max(courants), + 'accuracy_maxima': accuracy_maxima} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--artifacts', required=True) + parser.add_argument('--project') + parser.add_argument('--threads', type=int, default=8) + parser.add_argument('--repetitions', type=int, default=1) + parser.add_argument('--report') + args = parser.parse_args() + result = audit(args.artifacts, args.threads, args.repetitions, args.project) + encoded = json.dumps(result, indent=2, allow_nan=False)+'\n' + if args.report: + Path(args.report).write_text(encoded) + print(encoded, end='') + + +if __name__ == '__main__': + main() diff --git a/tests/test_access.py b/tests/test_access.py index 55f667b3..0c741b05 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -31,6 +31,17 @@ def owner(self, db=None): session = access.exchange(login) return db, access, session['token'], login + def test_local_existing_owner_mode_never_bootstraps(self): + db=self.catalog(version=8); access=Access(db,db.path) + with self.assertRaises(AccessError): access._local_owner_token(require_existing=True) + self.assertEqual(db.execute('SELECT count(*) FROM access_actors').fetchone()[0],0) + access._local_owner_token(initial=True) + login=access._local_owner_token(require_existing=True) + session=access.exchange(login) + with access.bind(session['token']): + access.require('admin','runtime_snapshots.create',owner_only=True) + access.logout() + def test_missing_actor_wrong_catalog_nested_reset_and_thread_isolation(self): db, access, token, _ = self.owner() with self.assertRaises(AccessError): diff --git a/tests/test_cpu_recipe_contract.py b/tests/test_cpu_recipe_contract.py new file mode 100644 index 00000000..8d4bcb77 --- /dev/null +++ b/tests/test_cpu_recipe_contract.py @@ -0,0 +1,106 @@ +"""Captured CPU material binding; fixtures qualify contracts, not model behavior.""" +import copy +import hashlib +import json +import unittest +from unittest.mock import patch + +import recipe_contract as contract +import recipe_process +import test_recipe_contract as fixtures +from workbench import WorkbenchError + + +class CpuRecipeContract(unittest.TestCase): + setUp = fixtures.RecipePipelineTests.setUp + + def cpu_graph(self): + self.value.update(runtime=contract.CPU_PROFILE, runtime_snapshot_sha256='a'*64) + self.recipe_raw = json.dumps(self.value).encode() + (self.project/'recipe.json').write_bytes(self.recipe_raw) + self.body['expected_sha256'] = hashlib.sha256(self.recipe_raw).hexdigest() + return self.dataset.pipelines.create_external(self.body) + + def descriptor(self): + return {'schema_version':1, 'profile':contract.CPU_PROFILE, + 'catalog_id':self.dataset.db.dataset_id, + 'root':str(self.dataset.db.path/'runtime-snapshots'/('a'*64)), + 'manifest_sha256':'a'*64, 'manifest_bytes':1000, 'file_count':10, + 'total_bytes':10000, 'python_version':'3.12', + 'packages':{'numpy':'2.5.4', 'torch':'2.8.0+cpu'}, + 'launch':{'python':'python/bin/python3.12', 'loader':'system/lib/ld-linux-x86-64.so.2', + 'home':'python', 'stdlib':'python/lib/python3.12', + 'dynload':'python/lib/python3.12/lib-dynload', 'packages':'packages', + 'library_dirs':['system/lib']}} + + def test_cpu_profile_requires_exact_selector_and_reduces_argument_budget(self): + value = copy.deepcopy(self.value) + for mutate in (lambda v:v.update(runtime=contract.CPU_PROFILE), + lambda v:v.update(runtime_snapshot_sha256='a'*64), + lambda v:v.update(runtime=contract.CPU_PROFILE,runtime_snapshot_sha256='../bad'), + lambda v:v.update(runtime=contract.CPU_PROFILE,runtime_snapshot_sha256='a'*64, + arguments=[{'literal':'x'}]*57)): + changed=copy.deepcopy(value); mutate(changed) + with self.assertRaises(WorkbenchError): contract.decode_definition(json.dumps(changed).encode()) + value.update(runtime=contract.CPU_PROFILE,runtime_snapshot_sha256='a'*64, + arguments=[{'literal':'x'}]*56) + self.assertEqual(contract.decode_definition(json.dumps(value).encode()),value) + + def test_cpu_capture_binds_snapshot_bootstrap_and_retains_cold_identity(self): + graph=self.cpu_graph(); descriptor=self.descriptor() + with patch('runtime_snapshots.RuntimeSnapshots.get',return_value=descriptor) as get: + with patch('subprocess.Popen',side_effect=AssertionError('Preflight executed code')): + preflight=self.dataset.pipelines.preflight_external(graph['id'],1) + captured=self.dataset.pipelines.capture_external(graph['id'],1,preflight['material_sha256'],self.root/'capture') + self.assertEqual(get.call_count,2) + material=captured['material']; spec=captured['spec'] + self.assertEqual(material['cpu_snapshot'],descriptor) + self.assertEqual(spec['cpu_snapshot'],descriptor) + self.assertEqual(spec['argv'][1:4],['-I','-S','-B']) + self.assertEqual(spec['argv'][5],'--cpu-launch') + self.assertEqual(spec['argv'][7],str(self.root/'capture/source/main.py')) + self.assertEqual(spec['native_loader_policy'],contract.NATIVE_POLICY) + self.assertEqual((self.root/'capture'/contract.CPU_BOOTSTRAP).read_bytes(),recipe_process.SOURCE) + self.assertEqual(json.loads((self.root/'capture'/contract.CPU_DESCRIPTOR).read_bytes()),descriptor) + full=dict(spec,attempt_id='b'*32,request_sha256='c'*64) + recipe_process.validate_spec(full) + # Retained helper/source identity is historical, never recomputed from a + # changed installation merely to inspect an old completed Run. + with patch.object(recipe_process,'SOURCE',b'new helper'), patch.object(contract,'runtime_identity',side_effect=AssertionError('Cold runtime access')): + contract.validate_spec(material,spec,self.root/'capture') + for mutate in ( + lambda m:m['cpu_snapshot'].update(catalog_id='f'*32), + lambda m:m['cpu_snapshot'].update(manifest_sha256='f'*64), + lambda m:m['cpu_snapshot'].update(root='/tmp/runtime-snapshots/'+'a'*64), + lambda m:m['captures'][contract.CPU_BOOTSTRAP].update(sha256='f'*64), + lambda m:m['captures'][contract.CPU_DESCRIPTOR].update(sha256='f'*64), + lambda m:m.pop('cpu_snapshot'), + lambda m:m['recipe'].update(runtime='system_python'), + ): + changed=copy.deepcopy(material);mutate(changed) + with self.assertRaises(WorkbenchError): contract.validate_material(changed) + + def test_omitted_cpu_profile_has_no_added_fields(self): + graph=self.dataset.pipelines.create_external(self.body) + preflight=self.dataset.pipelines.preflight_external(graph['id'],1) + material=preflight['material'];spec=contract.spec_for_material(material,self.root/'old') + self.assertNotIn('cpu_snapshot',material);self.assertNotIn('cpu_snapshot',spec) + self.assertNotIn('native_loader_policy',spec) + self.assertFalse(any(p.startswith('runtime-bootstrap/') for p in material['captures'])) + changed=copy.deepcopy(material);changed['cpu_snapshot']=self.descriptor() + with self.assertRaises(WorkbenchError):contract.validate_material(changed) + + def test_unicode_descriptor_uses_the_helpers_canonical_bytes(self): + graph=self.cpu_graph();descriptor=self.descriptor() + descriptor['root']=str(self.root/'κατάλογος'/'runtime-snapshots'/('a'*64)) + with patch('runtime_snapshots.RuntimeSnapshots.get',return_value=descriptor): + preflight=self.dataset.pipelines.preflight_external(graph['id'],1) + captured=self.dataset.pipelines.capture_external(graph['id'],1,preflight['material_sha256'],self.root/'λήψη') + raw=(self.root/'λήψη'/contract.CPU_DESCRIPTOR).read_bytes() + self.assertEqual(raw,recipe_process.encoded(descriptor)) + self.assertEqual(json.loads(raw),descriptor) + recipe_process.validate_spec(dict(captured['spec'],attempt_id='b'*32,request_sha256='c'*64)) + contract.validate_spec(captured['material'],captured['spec'],self.root/'λήψη') + + +if __name__ == '__main__': unittest.main() diff --git a/tests/test_recipe_process.py b/tests/test_recipe_process.py index 897d6371..7614d600 100644 --- a/tests/test_recipe_process.py +++ b/tests/test_recipe_process.py @@ -206,3 +206,178 @@ def test_closed_controller_channel_reports_lost_once_and_cancel_is_idempotent(se self.assertEqual(handle.poll(),[{'type':'lost','reason':'custodian_channel_closed'}]) self.assertEqual(handle.poll(),[]) self.assertTrue(handle.close_control()['drained']) + + +class CpuSnapshotContractTests(unittest.TestCase): + """Authored tiny inventories prove custody checks, not package acceptance.""" + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.base=Path(self.tmp.name) + contents={ + 'python/bin/python3.12':b'\x7fELFauthored interpreter, not executable', + 'system/lib/ld-linux-x86-64.so.2':b'\x7fELFauthored loader, not executable', + 'python/lib/python3.12/os.py':b'# authored stdlib fixture\n', + 'python/lib/python3.12/lib-dynload/_fixture.so':b'\x7fELFauthored extension', + 'packages/numpy/__init__.py':b'# authored package fixture\n', + } + self.manifest={'schema_version':1,'profile':custody.CPU_PROFILE,'python_version':'3.12', + 'packages':{'numpy':'fixture','torch':'fixture'}, + 'launch':{'python':'python/bin/python3.12','loader':'system/lib/ld-linux-x86-64.so.2', + 'home':'python','stdlib':'python/lib/python3.12','dynload':'python/lib/python3.12/lib-dynload', + 'packages':'packages','library_dirs':['system/lib']}, + 'files':[{'path':name,'bytes':len(raw),'sha256':custody.digest(raw), + 'mode':0o500 if name in ('python/bin/python3.12','system/lib/ld-linux-x86-64.so.2') else 0o400, + 'kind':'elf' if raw.startswith(b'\x7fELF') else 'data'} for name,raw in sorted(contents.items())], + 'total_bytes':sum(map(len,contents.values()))} + raw=custody.encoded(self.manifest);self.root=self.base/'runtime-snapshots'/custody.digest(raw) + self.root.mkdir(parents=True,mode=0o700) + for item in self.manifest['files']: + target=self.root/item['path'];target.parent.mkdir(parents=True,exist_ok=True) + target.write_bytes(contents[item['path']]);target.chmod(item['mode']) + (self.root/'manifest.json').write_bytes(raw);(self.root/'manifest.json').chmod(0o400) + self.descriptor={'schema_version':1,'profile':custody.CPU_PROFILE,'catalog_id':'a'*32, + 'root':str(self.root),'manifest_sha256':custody.digest(raw),'manifest_bytes':len(raw), + 'file_count':len(contents),'total_bytes':self.manifest['total_bytes'],'python_version':'3.12', + 'packages':self.manifest['packages'],'launch':self.manifest['launch']} + + def test_virtual_torch_origins_are_exact_and_captured(self): + import sys + import types + defining=types.ModuleType('torch._ops') + defining.__file__=str(self.root/'packages/torch/_ops.py') + cls=type('_Ops',(types.ModuleType,),{'__module__':'torch._ops','__file__':'_ops.py'}) + defining._Ops=cls + module=cls('torch.ops') + modules={'torch._ops':defining,'torch.ops':module} + manifest={**self.manifest,'files':[*self.manifest['files'],{'path':'packages/torch/_ops.py'}]} + paths=[str(self.root/manifest['launch'][key]) for key in ('stdlib','dynload','packages')] + def check(): + with patch.object(sys,'modules',modules),patch.object(sys,'path',paths): + return custody._cpu_origins(manifest,self.descriptor,'/captured/bootstrap.py') + self.assertEqual(check()['torch.ops']['defining_path'],defining.__file__) + modules['other.ops']=modules.pop('torch.ops') + with self.assertRaises(custody.ProcessError):check() + modules['torch.ops']=modules.pop('other.ops') + modules['torch.ops']=types.ModuleType('torch.ops');modules['torch.ops'].__file__='_ops.py' + with self.assertRaises(custody.ProcessError):check() + modules['torch.ops']=module;module.__file__='other.py' + with self.assertRaises(custody.ProcessError):check() + module.__file__='_ops.py';defining.__file__='/outside/_ops.py' + with self.assertRaises(custody.ProcessError):check() + + def test_canonical_roundtrip_without_live_packages(self): + self.assertEqual(custody.validate_cpu_snapshot_descriptor(self.descriptor),self.descriptor) + self.assertEqual(custody.verify_cpu_snapshot(self.descriptor),self.manifest) + + def test_pure_bounds_paths_modes_and_descriptor_binding(self): + import copy + for field,value in [('file_count',40001),('total_bytes',custody.CPU_TOTAL_LIMIT+1), + ('manifest_bytes',True),('root',str(self.base/'elsewhere')),('catalog_id','x')]: + altered=copy.deepcopy(self.descriptor);altered[field]=value + with self.subTest(field=field),self.assertRaises(custody.ProcessError):custody.validate_cpu_snapshot_descriptor(altered) + for field,value in [('path','../escape'),('mode',0o777),('bytes',custody.CPU_FILE_LIMIT+1),('kind','symlink')]: + altered=copy.deepcopy(self.manifest);altered['files'][0][field]=value + with self.subTest(field=field),self.assertRaises(custody.ProcessError):custody.validate_cpu_manifest(altered) + altered=copy.deepcopy(self.manifest);altered['launch']['library_dirs']=['system/lib:/usr/lib'] + with self.assertRaises(custody.ProcessError):custody.validate_cpu_manifest(altered) + altered=copy.deepcopy(self.descriptor);altered['packages']['numpy']='changed' + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(altered) + + def test_mutated_bytes_mode_missing_and_unlisted_refused(self): + target=self.root/'packages/numpy/__init__.py';raw=target.read_bytes() + target.chmod(0o600);target.write_bytes(b'X'*len(raw));target.chmod(0o400) + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + target.chmod(0o600);target.write_bytes(raw) + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + target.chmod(0o400);extra=self.root/'packages/unauthorized.py';extra.write_text('raise RuntimeError()') + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + extra.unlink();target.unlink() + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + + def test_symlink_and_hardlink_file_custody_refused(self): + target=self.root/'packages/numpy/__init__.py';raw=target.read_bytes();target.unlink() + outside=self.base/'outside';outside.write_bytes(raw);outside.chmod(0o400);target.symlink_to(outside) + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + target.unlink();os.link(outside,target) + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + + def test_manifest_change_and_kind_mismatch_refused(self): + target=self.root/'manifest.json';target.chmod(0o600);target.write_bytes(target.read_bytes()+b'\n');target.chmod(0o400) + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + + def cpu_spec(self): + executable=str(Path('/usr/bin/python3').resolve());entry=self.base/'recipe.py';entry.write_text('pass\n') + bootstrap=self.base/'recipe_process.py';bootstrap.write_bytes(custody.SOURCE) + descriptor=self.base/'snapshot.json';descriptor.write_bytes(custody.encoded(self.descriptor)) + spec={'schema_version':1,'attempt_id':'b'*32,'request_sha256':'c'*64, + 'argv':[executable,'-I','-S','-B',str(bootstrap),'--cpu-launch',str(descriptor),str(entry)], + 'env':{},'cwd':str(self.base),'stdin_base64':'','wall_seconds':5,'executable_sha256':custody.digest(Path(executable).read_bytes()), + 'max_output_bytes':1024,'output_dir':str(self.base),'runtime_files':[], + 'trusted_install_owner_uid':os.stat('/usr').st_uid,'native_loader_policy':'glibc_x86_64_no_preload_v1', + 'cpu_snapshot':self.descriptor,'captured_files':[{'path':str(path),'bytes':path.stat().st_size,'sha256':custody.digest(path.read_bytes())} for path in (bootstrap,descriptor,entry)]} + return spec + + def test_cpu_launch_vector_requires_bound_descriptor(self): + spec=self.cpu_spec() + self.assertEqual(custody.validate_spec(spec),spec) + spec['captured_files'][1]['sha256']='d'*64 + with self.assertRaises(custody.ProcessError):custody.validate_spec(spec) + + def test_directory_and_preparation_budget_are_bounded(self): + (self.root/'empty-unlisted').mkdir() + with self.assertRaises(custody.ProcessError):custody.verify_cpu_snapshot(self.descriptor) + (self.root/'empty-unlisted').rmdir() + with patch.object(custody.time,'monotonic',side_effect=[0,61]): + with self.assertRaisesRegex(custody.ProcessError,'deadline'): + custody.verify_cpu_snapshot(self.descriptor) + import copy + altered=copy.deepcopy(self.manifest) + altered['files'][0]['path']='/'.join(['deep']*33) + with self.assertRaises(custody.ProcessError):custody.validate_cpu_manifest(altered) + + def test_cancel_during_delayed_go_verification_drains_before_fork(self): + # Fault-injected verification latency, real helper/control/drain. The tiny + # authored ELF files never execute and do not qualify a CPU runtime. + injected=b""" +_original_verify_cpu_snapshot = verify_cpu_snapshot +def verify_cpu_snapshot(descriptor, *, checkpoint=None): + if checkpoint is not None: + (Path(descriptor['root']).parent.parent/'verification-entered').write_text('entered') + for _ in range(500): + checkpoint() + time.sleep(.01) + return _original_verify_cpu_snapshot(descriptor, checkpoint=checkpoint) + +""" + source=custody.SOURCE.replace(b"if __name__=='__main__':",injected+b"if __name__=='__main__':") + with patch.object(custody,'SOURCE',source): + handle=custody.launch(self.cpu_spec(),self.base/'custody') + try: + deadline=time.monotonic()+8 + while not handle.ready and time.monotonic()