diff --git a/README.md b/README.md index 9d235c45..862dd154 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,9 @@ login file: `.venv/bin/python access.py login --data ./data --out /private/new-l Use an existing private directory and a new filename, then enter its code in the browser's Session dialog. Codes expire after ten minutes; sessions last eight hours. Localhost is not an authenticated identity. Existing catalogs require the -explicit schema-7 upgrade before starting this version. See [actor access](docs/contracts/actors.md) +explicit schema-8 upgrade before starting this version. See [actor access](docs/contracts/actors.md) for scoped agent credentials and [storage upgrades](docs/contracts/workspace-storage.md). -Existing execution remains owner-only; agent execution approval is a later stage. +Existing CPU and integrity execution remains owner-only. Scoped agents can submit external recipes with an exact, one-use owner approval; see [external recipes](docs/contracts/external-recipes.md). The setup script uses a Unix socket under `~/.local/share/tuldok/postgres`, generates private credentials, and creates a database role without superuser, @@ -882,4 +882,16 @@ Start with `python app.py --data ./catalog --project-root research=/absolute/res ## Scientific field sequences -For a conforming [scientific field bundle](docs/contracts/scientific-fields.md), use **Import fields** on its project Data reference. Inspect frames, fields and index slices with physical coordinates, units and producer stage labels; save whole-trajectory review and freeze a canonical release. Original bundle and floating-point bytes remain unchanged. First-observed-state and declared-family grouping does not certify independent initial conditions. The cold reader `python scientific_release.py RELEASE.zip` verifies frozen custody; generation and training require later execution milestones. +For a conforming [scientific field bundle](docs/contracts/scientific-fields.md), use **Import fields** on its project Data reference. Inspect frames, fields and index slices with physical coordinates, units and producer stage labels; save whole-trajectory review and freeze a canonical release. Original bundle and floating-point bytes remain unchanged. First-observed-state and declared-family grouping does not certify independent initial conditions. The cold reader `python scientific_release.py RELEASE.zip` verifies frozen custody; the optional [Rheon producer](adapters/rheon/README.md) adds bounded native generation, while training and autonomous evaluation remain later milestones. + + +## Optional native generation + +The [Rheon adapter](adapters/rheon/README.md) assembles a project from an explicitly +supplied qualified binary and original build receipt. Open that project, edit the +bounded grid/frame/seed controls in Pipeline, review the exact execution material +and submit. On a completed scientific artifact, **Import into Data** creates a +draft for inspection, human review and freeze. Native bytes, loaded-library +identity and the original custody receipt are retained. Initial support is Jacobi +generation at grids 8–32 and one or two frames; this is not training, GPU or +general physics acceptance. Generic projects and Data work without the adapter. diff --git a/adapters/rheon/README.md b/adapters/rheon/README.md new file mode 100644 index 00000000..ff9ce46c --- /dev/null +++ b/adapters/rheon/README.md @@ -0,0 +1,48 @@ +# Optional Rheon producer + +This package wraps one pinned native trajectory probe as an ordinary Tuldok external +recipe. Tuldok's generic project, data, approval, and Runs owners do not import it. +Removing this package does not disable generic projects or scientific data. + +Assemble a new local project from an existing qualified binary and its original +build/capture receipt: + +```sh +python adapters/rheon/project.py \ + --binary /absolute/path/rheon-trajectory-probe \ + --build-receipt /absolute/path/original-receipt.json \ + --destination /absolute/path/new-project +``` + +The assembler creates only the new destination. It never builds, executes, +changes the original files, or overwrites an existing destination. It copies the +original receipt unchanged; that receipt supplies build evidence, not a claim +that earlier captured data was generated by the new Run. The receipt must bind +this adapter's pinned binary, lab/Rheon commits, locked build and toolchain. + +Configure the new directory as an allowed project root, open `project.json`, and +select its generation recipe. Bind the `build_receipt` input to the project's +`build_receipt` data reference. The existing Pipeline controls expose grid 8–32, +frames 1–2, and seed 0–2147483647. Defaults are 32, 2, and 100. Jacobi, timestep +1/60, density 1.25, and native solver settings are fixed. Learned pressure and +SGS are not offered by this adapter. + +Review the normal execution preflight and approve its exact material before +running. The generic host captures source, native policy, and inputs; the source +binary remains mode 0400. The captured wrapper seals the approved native and +loader bytes, observes loaded file identities while native READY waits for the +first command, and requests complete frames. Runs owns cancellation, descendants, +durable completion, and publication. The adapter is trusted local code, not a +filesystem or network sandbox. + +A successful Run publishes `fields.zip` (role `scientific_fields`), `wire.bin`, +`producer-receipt.json`, and `native-diagnostics.txt`. Import `fields.zip` into +Data through its verified artifact action, inspect fields/frames, then separately +review and freeze it. Import parses the actual bytes; the role is only a UI hint. +The six fields preserve little-endian f32/f64 values, C-order ZYX and MAC face +staggering. Native units remain unspecified; field stages are distinct. + +See [the producer contract](../../docs/contracts/rheon-producer.md). Local tests +run with `python -m unittest discover -s tests -p test_rheon_adapter.py -v`. +The two actual-native tests explicitly skip when the separately provisioned +pinned scratch build is absent. Authored parser fixtures are not native acceptance. diff --git a/adapters/rheon/generate.py b/adapters/rheon/generate.py new file mode 100644 index 00000000..2439bc63 --- /dev/null +++ b/adapters/rheon/generate.py @@ -0,0 +1,338 @@ +"""Optional pinned Rheon adapter. Runs, not this adapter, owns process custody.""" +import argparse +import fcntl +import hashlib +import io +import json +import math +import os +from pathlib import Path +import re +import selectors +import stat +import struct +import subprocess +import time +import zipfile + +BINARY_SHA256 = '4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff' +SOURCE_PINS = {'lab': 'e8a90bfef6b2955c502008ccc62443e29dd9f2ec', + 'Rheon': '2209012bc5f902693d0f695ec1c2d854126af317'} +LIBRARY_DIRECTORY = '/usr/lib/x86_64-linux-gnu' +LOADER_ARGUMENTS = ['--inhibit-cache', '--library-path', LIBRARY_DIRECTORY, '--glibc-hwcaps-mask', ''] +WIRE_LIMIT = 4 * 1024 * 1024 +DIAGNOSTIC_LIMIT = 65536 +HASH = re.compile(r'^[a-f0-9]{64}$') + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def encode(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def decode(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON key.') + result[key] = value + return result + try: + return json.loads(raw, object_pairs_hook=pairs, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Nonfinite JSON.'))) + except (UnicodeError, RecursionError) as error: + raise ValueError('Invalid bounded JSON.') from error + + +def closed(value, keys): + if type(value) is not dict or set(value) != set(keys): + raise ValueError('Unexpected contract fields.') + + +def controls(grid, frames, seed): + for value, low, high in ((grid, 8, 32), (frames, 1, 2), (seed, 0, 2147483647)): + if type(value) is not int or not low <= value <= high: + raise ValueError('Supported controls: grid 8–32, frames 1–2, seed 0–2147483647; Jacobi only.') + return {'grid': grid, 'frames': frames, 'seed': seed, 'method': 'jacobi'} + + +def capture(path, limit): + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, 'rb') as handle: + before = os.fstat(handle.fileno()) + if not stat.S_ISREG(before.st_mode) or before.st_size > limit: + raise ValueError('Expected a bounded regular input.') + raw = handle.read(limit + 1) + after = os.fstat(handle.fileno()) + if len(raw) > limit or (before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns): + raise ValueError('Input changed during capture.') + return raw, after + + +def build_receipt(raw, native): + if len(raw) > DIAGNOSTIC_LIMIT: + raise ValueError('Build receipt exceeds 64 KiB.') + value = decode(raw) + if type(value) is not dict or value.get('binary_sha256') != sha(native) or sha(native) != BINARY_SHA256: + raise ValueError('Unqualified native binary/build receipt.') + sources = value.get('source') + if type(sources) is not dict or any(type(sources.get(name)) is not dict or sources[name].get('commit') != pin for name, pin in SOURCE_PINS.items()): + raise ValueError('Native source pins differ.') + if not all(type(value.get(key)) is str and value[key] for key in ('rustc', 'cargo', 'toolchain_archive_sha256')): + raise ValueError('Original toolchain receipt is required.') + if type(value.get('build')) is not dict or value['build'].get('command') != 'cargo build --release --locked --manifest-path probes/trajectory/Cargo.toml': + raise ValueError('Original locked build receipt is required.') + return value + + +def native_policy(raw, native): + if len(raw) > 1024 * 1024: + raise ValueError('Native policy is oversized.') + value = decode(raw) + closed(value, ('schema_version', 'policy', 'source', 'loader', 'library_directory', 'loader_arguments', 'runtime_files', 'absent_paths')) + if type(value['schema_version']) is not int or value['schema_version'] != 1 or value['policy'] != 'glibc_x86_64_no_preload_v1': + raise ValueError('Unsupported native policy.') + closed(value['source'], ('path', 'sha256')) + if value['source']['sha256'] != sha(native) or type(value['source']['path']) is not str: + raise ValueError('Native source differs from approved policy.') + if value['library_directory'] != LIBRARY_DIRECTORY or value['loader_arguments'] != LOADER_ARGUMENTS or value['absent_paths'] != ['/etc/ld.so.preload']: + raise ValueError('Unsupported native loader policy.') + if os.path.lexists('/etc/ld.so.preload'): + raise ValueError('System preload is unsupported.') + if type(value['runtime_files']) is not list or not 1 <= len(value['runtime_files']) <= 32: + raise ValueError('Invalid native closure.') + observed = {} + for item in value['runtime_files']: + closed(item, ('path', 'bytes', 'sha256')) + if type(item['path']) is not str or Path(item['path']).parent != Path(LIBRARY_DIRECTORY) or item['path'] in observed: + raise ValueError('Unexpected native dependency path.') + if type(item['bytes']) is not int or not 0 < item['bytes'] <= 8 * 1024 * 1024 or type(item['sha256']) is not str or not HASH.fullmatch(item['sha256']): + raise ValueError('Invalid native dependency identity.') + data, info = capture(item['path'], item['bytes']) + if len(data) != item['bytes'] or sha(data) != item['sha256']: + raise ValueError('Native runtime changed.') + observed[item['path']] = (data, info) + closed(value['loader'], ('path', 'bytes', 'sha256')) + loader = value['loader'] + if loader['path'] != LIBRARY_DIRECTORY + '/ld-linux-x86-64.so.2' or loader not in value['runtime_files']: + raise ValueError('Loader is outside approved closure.') + return value, observed + + +def seal(name, raw): + descriptor = os.memfd_create(name, os.MFD_ALLOW_SEALING) + try: + with os.fdopen(os.dup(descriptor), 'wb') as handle: + handle.write(raw) + os.lseek(descriptor, 0, os.SEEK_SET) + # Linux UAPI; Python builds can omit these symbolic constants. + fcntl.fcntl(descriptor, getattr(fcntl, 'F_ADD_SEALS', 1033), 0x000f) + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def mapping_evidence(pid, observed, native_fd, loader_fd): + expected = {(info.st_dev, info.st_ino): path for path, (_raw, info) in observed.items()} + for name, descriptor in (('sealed-native', native_fd), ('sealed-loader', loader_fd)): + info = os.fstat(descriptor) + expected[(info.st_dev, info.st_ino)] = name + with Path(f'/proc/{pid}/maps').open('rb') as handle: + raw = handle.read(1024 * 1024 + 1) + if len(raw) > 1024 * 1024: + raise ValueError('Native mapping inventory exceeds bound.') + found = {} + for line in raw.decode().splitlines(): + fields = line.split(maxsplit=5) + inode = int(fields[4]) + if not inode: + continue # Anonymous memory and kernel vDSO/vvar mappings. + major, minor = (int(part, 16) for part in fields[3].split(':')) + identity = (os.makedev(major, minor), inode) + if identity not in expected: + raise ValueError('Native loaded an undeclared file mapping.') + name = expected[identity] + found[name] = {'device': fields[3], 'inode': inode} + required = set(observed) - {LIBRARY_DIRECTORY + '/ld-linux-x86-64.so.2'} + if not required | {'sealed-native', 'sealed-loader'} <= set(found): + raise ValueError('Native mapping inventory is incomplete.') + return found + + +def validate_ready(ready, request): + if type(ready) is not dict: raise ValueError('Native READY must be an object.') + n = request['grid'] + if (ready.get('schema') != 'rheon-trajectory-v1' or ready.get('n') != n or ready.get('method') != 'jacobi' + or ready.get('seed') != request['seed'] or ready.get('dt') != 1 / 60 or ready.get('density') != 1.25 + or ready.get('spacing') != [1 / n] * 3 or ready.get('face_lengths') != [(n+1)*n*n]*3): + raise ValueError('Native ready controls differ.') + +def decode_wire(raw, request): + if len(raw) > WIRE_LIMIT: + raise ValueError('Native wire exceeds bound.') + position = 0 + def take(count): + nonlocal position + if position + count > len(raw): + raise ValueError('Truncated native wire.') + value = raw[position:position + count]; position += count + return value + def message(): + length, = struct.unpack(' DIAGNOSTIC_LIMIT: + raise ValueError('Native message exceeds bound.') + value = decode(take(length)) + if type(value) is not dict: + raise ValueError('Native message must be an object.') + return value + ready = message(); n = request['grid'] + validate_ready(ready, request) + fields = [('rhs', ' 1e12 or not math.isfinite(value) for value in report.values()): + raise ValueError('Native report is nonfinite or malformed.') + if not 0 <= report.get('actual_divergence_max', math.inf) <= 1e-5 or not 0 <= report.get('courant', math.inf) <= 1: + raise ValueError('Native accepted-step diagnostics failed.') + arrays = [] + for name, dtype, center, stage in fields: + shape = [n, n, n] + if center != 'cell': shape[{'face_x': 2, 'face_y': 1, 'face_z': 0}[center]] += 1 + data = take(math.prod(shape) * (8 if dtype == '= deadline: raise ValueError('Native observation deadline exceeded.') + for key, _mask in selector.select(.1): + raw = os.read(key.fd, 65536) + if not raw: selector.unregister(key.fileobj); continue + target, limit = (wire, WIRE_LIMIT) if key.data == 'wire' else (diagnostics, DIAGNOSTIC_LIMIT) + if len(target) + len(raw) > limit: raise ValueError('Native stream exceeds bound.') + target.extend(raw) + if not sent and len(wire) >= 4: + length, = struct.unpack(' DIAGNOSTIC_LIMIT: raise ValueError('Native READY exceeds bound.') + if len(wire) >= length + 4: + # Native waits for our first frame command, so maps are observed + # before any generated state. No PID can be recycled before wait. + validate_ready(decode(wire[4:4+length]), request) + mappings = mapping_evidence(child.pid, observed, *descriptors) + child.stdin.write(bytes([2] * request['frames'] + [0])); child.stdin.close(); sent = True + if child.wait(timeout=3) != 0 or not sent: raise ValueError('Native generation failed.') + decode_wire(bytes(wire), request) + return bytes(wire), bytes(diagnostics), mappings + finally: + selector.close() + if child is not None: + if child.poll() is None: + child.kill() + child.wait(timeout=3) + for stream in (child.stdin, child.stdout, child.stderr): + if stream is not None and not stream.closed: stream.close() + for descriptor in descriptors: os.close(descriptor) + + +def generate(native_path, build_path, policy_path, grid, frames, seed, output): + request = controls(grid, frames, seed) + native, _ = capture(native_path, 8 * 1024 * 1024) + build_raw, _ = capture(build_path, DIAGNOSTIC_LIMIT); build_receipt(build_raw, native) + policy_raw, _ = capture(policy_path, 1024 * 1024); policy, observed = native_policy(policy_raw, native) + output = Path(output) + if output.is_symlink() or not output.is_dir() or any(output.iterdir()): raise ValueError('Output must be an empty existing directory.') + wire, diagnostics, mappings = observe(native, policy, observed, request) + ready, decoded = decode_wire(wire, request) + receipt = {'schema_version': 1, 'producer': 'rheon-trajectory-probe', 'controls': request, + 'binary_sha256': sha(native), 'build_receipt_sha256': sha(build_raw), 'native_policy_sha256': sha(policy_raw), + 'wrapper_sha256': sha(Path(__file__).read_bytes()), 'stdin_hex': bytes([2]*frames+[0]).hex(), + 'wire_sha256': sha(wire), 'wire_bytes': len(wire), 'diagnostics_sha256': sha(diagnostics), + 'loaded_mappings': mappings, 'ready': ready, 'frames': [frame['report'] for frame in decoded], + 'limitations': ['Native units unspecified; algorithm stages differ.', 'Generated observation, not training, historical reproduction or general physics acceptance.']} + receipt_raw = encode(receipt) + bundle = scientific_bundle(wire, request, build_raw, receipt_raw, policy_raw) + outputs = {'fields.zip': bundle, 'wire.bin': wire, 'producer-receipt.json': receipt_raw, 'native-diagnostics.txt': diagnostics} + if sum(map(len, outputs.values())) > 16 * 1024 * 1024: raise ValueError('Output budget exceeded.') + for name, data in outputs.items(): + with (output/name).open('xb') as handle: handle.write(data); handle.flush(); os.fsync(handle.fileno()) + print(json.dumps({'stage': 'generated', 'frames': frames, 'fields': 6, 'bundle_bytes': len(bundle)}), flush=True) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('native', 'build-receipt', 'native-policy', 'output'): parser.add_argument('--'+name, required=True) + parser.add_argument('--grid', type=int, required=True); parser.add_argument('--frames', type=int, required=True); parser.add_argument('--seed', type=int, required=True) + args = parser.parse_args() + generate(args.native, args.build_receipt, args.native_policy, args.grid, args.frames, args.seed, args.output) + + +if __name__ == '__main__': main() diff --git a/adapters/rheon/project.py b/adapters/rheon/project.py new file mode 100644 index 00000000..9d6e858f --- /dev/null +++ b/adapters/rheon/project.py @@ -0,0 +1,71 @@ +"""Assemble a new explicit project from an existing pinned native build receipt.""" +import argparse +import os +from pathlib import Path +import shutil +import uuid + +if __package__: + from . import generate +else: + import generate + + +def definition(wrapper, native): + return {'schema_version': 1, 'format': 'external_recipe_v1', 'id': 'rheon-generate', + 'name': 'Rheon field generation', 'version': 1, 'runtime': 'system_python', + 'sources': [{'path': 'generate.py', 'sha256': generate.sha(wrapper)}, + {'path': 'rheon-trajectory-probe', 'sha256': generate.sha(native)}], + 'native_sources': ['rheon-trajectory-probe'], 'entrypoint': 'generate.py', + 'arguments': [{'literal': '--native'}, {'source': 'rheon-trajectory-probe'}, + {'literal': '--build-receipt'}, {'input': 'build_receipt'}, + {'literal': '--native-policy'}, {'native_policy': 'rheon-trajectory-probe'}, + {'literal': '--grid'}, {'parameter': 'grid'}, {'literal': '--frames'}, {'parameter': 'frames'}, + {'literal': '--seed'}, {'parameter': 'seed'}, {'literal': '--output'}, {'output_dir': True}], + 'parameters': {'grid': {'type': 'integer', 'min': 8, 'max': 32, 'default': 32}, + 'frames': {'type': 'integer', 'min': 1, 'max': 2, 'default': 2}, + 'seed': {'type': 'integer', 'min': 0, 'max': 2147483647, 'default': 100}}, + 'inputs': {'build_receipt': {'max_bytes': generate.DIAGNOSTIC_LIMIT}}, + 'outputs': [{'path': name, 'role': role, 'max_bytes': limit, 'required': True} for name, role, limit in ( + ('fields.zip', 'scientific_fields', 8*1024*1024), ('wire.bin', 'native_wire', generate.WIRE_LIMIT), + ('producer-receipt.json', 'producer_receipt', generate.DIAGNOSTIC_LIMIT), + ('native-diagnostics.txt', 'native_diagnostics', generate.DIAGNOSTIC_LIMIT))], + 'environment': {}, 'resources': {'wall_seconds': 120, 'max_output_bytes': 16*1024*1024}} + + +def assemble(binary, receipt, destination): + native, _ = generate.capture(binary, 8*1024*1024) + build, _ = generate.capture(receipt, generate.DIAGNOSTIC_LIMIT) + generate.build_receipt(build, native) + wrapper = Path(__file__).with_name('generate.py').read_bytes() + recipe = generate.encode(definition(wrapper, native)) + project = {'schema_version': 1, 'id': uuid.uuid4().hex, 'name': 'Rheon bounded field generation', + 'data': [{'id': 'build_receipt', 'label': 'Original native build evidence', + 'path': 'build-receipt.json', 'sha256': generate.sha(build)}], + 'recipes': [{'id': 'generate', 'label': 'Generate Jacobi fields', 'path': 'recipe.json', 'sha256': generate.sha(recipe)}], + 'evidence': [], 'requirements': [{'id': 'native-runtime', 'label': 'Qualified Linux x86-64 glibc runtime; trusted local code'}]} + destination = Path(destination) + destination.mkdir(mode=0o700, exist_ok=False) + try: + for name, raw in {'generate.py': wrapper, 'rheon-trajectory-probe': native, 'build-receipt.json': build, + 'recipe.json': recipe, 'project.json': generate.encode(project)}.items(): + with (destination/name).open('xb') as handle: + handle.write(raw); handle.flush(); os.fchmod(handle.fileno(), 0o400); os.fsync(handle.fileno()) + descriptor = os.open(destination, os.O_RDONLY | os.O_DIRECTORY) + try: os.fsync(descriptor) + finally: os.close(descriptor) + except BaseException: + shutil.rmtree(destination) + raise + return project + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('binary', 'build-receipt', 'destination'): parser.add_argument('--'+name, required=True) + args = parser.parse_args() + project = assemble(args.binary, args.build_receipt, args.destination) + print(generate.encode({'project_id': project['id'], 'manifest': str(Path(args.destination)/'project.json')}).decode()) + + +if __name__ == '__main__': main() diff --git a/app.py b/app.py index 6c217636..30831584 100644 --- a/app.py +++ b/app.py @@ -757,7 +757,7 @@ def _post(self): if not 0 < length <= limit: raise ValueError('Request is too large or empty.') raw_body = self.rfile.read(length) - body = projects.decode(raw_body) if path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/') else spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) + body = projects.decode(raw_body) if (path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/') or (path.startswith('/api/workspace/runs/') and '/import-data/' in path)) else spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) if not isinstance(body, dict): raise ValueError('Expected an object.') path = urlsplit(self.path).path @@ -771,6 +771,11 @@ def _post(self): body['expected_sha256'], projects.DATA_LIMIT) admission = {key: value for key, value in body.items() if key != 'expected_sha256'} return self.reply(dataset.workbench.scientific.admit(raw, admission, origin), 201) + if len(parts) == 4 and parts[0] == 'runs' and parts[2] == 'import-data': + projects.closed(body, ('expected_sha256',), ('name', 'rights', 'groups', 'parents')) + admission = {key: value for key, value in body.items() if key != 'expected_sha256'} + return self.reply(dataset.workbench.scientific.admit_run_artifact( + parts[1], parts[3], body['expected_sha256'], admission), 201) if parts == ['graphs']: return self.reply(dataset.pipelines.create(body), 201) if parts == ['graphs', 'external']: diff --git a/docs/contracts/external-recipes.md b/docs/contracts/external-recipes.md index f38a8301..62b57824 100644 --- a/docs/contracts/external-recipes.md +++ b/docs/contracts/external-recipes.md @@ -131,3 +131,28 @@ Existing Run list/get/cancel/artifact routes serve these attempts. Classificatio predictions and test-exposure operations do not apply. Schema 8 adds local grants and immutable consumption records; explicit quiesced upgrade and versioned backup/restore retain schemas 1–7. + + +## Optional native dependencies + +An explicit `native_sources` list selects 1–8 paths already declared in `sources`. +A `{native_policy: source_path}` argument refers to host-generated captured JSON. +Omitting native declarations preserves existing definition/material/spec shapes. +The initial policy is `glibc_x86_64_no_preload_v1`: bounded ELF64 little-endian +x86-64 PIE with the qualified interpreter, recursive exact dependencies under +`/usr/lib/x86_64-linux-gnu`, no RPATH/RUNPATH, audit, filter or auxiliary loading. +The material binds loader/library identities and generated policy bytes. The +helper checks the fixed trusted `/etc` directory and absence of any +`/etc/ld.so.preload` entry at launch, READY and GO. + +The optional producer verifies and seals native and loader bytes, disables loader +cache and hardware-capability substitutions, and checks actual mapped file +identities before requesting work. This qualifies pinned trusted code; it is not +an arbitrary dynamic-plugin or hostile-administrator sandbox. See the optional +[Rheon producer](rheon-producer.md) for the bounded concrete workflow. + +Completed external artifacts can be captured with server-derived custody evidence +for [scientific Data import](scientific-fields.md). Restore preserves the original +absolute execution paths and spec hash as historical declarations. Artifact reads +use the restored catalog's owned paths. Dispatch refuses a staging-path mismatch; +restoration never replays a historical attempt. diff --git a/docs/contracts/rheon-producer.md b/docs/contracts/rheon-producer.md new file mode 100644 index 00000000..54f2aff3 --- /dev/null +++ b/docs/contracts/rheon-producer.md @@ -0,0 +1,101 @@ +# Optional Rheon field producer + +The optional `adapters/rheon` package implements the producer-specific boundary; +no generic owner requires Rheon. It uses the existing external recipe, approval, +sole Runs supervisor, scientific field format, and artifact import. It neither +registers a second scheduler nor modifies lab/Rheon algorithms. + +## Qualified source and controls + +Initial native SHA256 is +`4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff`. +Lab commit is `e8a90bfef6b2955c502008ccc62443e29dd9f2ec`; Rheon commit is +`2209012bc5f902693d0f695ec1c2d854126af317`. The original receipt must bind these +identities and `cargo build --release --locked --manifest-path probes/trajectory/Cargo.toml` +with toolchain evidence. Source receipt bytes are retained unchanged. + +Grid is an integer 8–32, frame count 1–2, and seed 0–2147483647. Jacobi is fixed. +Native timestep 1/60, density 1.25, solver thresholds, initial zero fields and +seeded forcing remain the original algorithm's constants. These are not editable +physical controls or claims of SI units. Defaults are 32/two frames/seed 100. +The recipe declares a 120-second wall bound and 16 MiB aggregate output bound. + +## Public adapter interface + +`project.assemble(binary, receipt, destination)` validates existing captured +inputs and creates a new directory containing `project.json`, `recipe.json`, +`generate.py`, `rheon-trajectory-probe`, and `build-receipt.json`. It returns the +project manifest, refuses overwrite, and never executes the producer. Inputs and +source are ordinary explicit project references; no invented input dataset is +introduced for the internally generated initial condition. + +The standalone captured wrapper accepts `--native`, `--build-receipt`, +`--native-policy`, `--grid`, `--frames`, `--seed`, and `--output`. +`generate(...)` validates all inputs before spawning and returns its producer +receipt only after successful native exit, complete wire validation, and writing +all declared outputs. It requires an empty existing output directory. Publication +into durable completed Runs remains the Runs owner's separate responsibility. + +## Native execution identity + +The generic host supplies a captured `glibc_x86_64_no_preload_v1` policy through +`{native_policy: sourcepath}`. It binds source, loader, native library inventory, +fixed arguments and preload absence. The wrapper reads and hashes bounded regular +files without following final symlinks, then seals the exact native and loader +bytes into memfds. It executes the loader descriptor with cache disabled, the +explicit `/usr/lib/x86_64-linux-gnu` library path, and an empty glibc hwcaps mask. +It never changes source mode or executes a pathname after hashing different bytes. + +At READY, before sending frame commands, the wrapper reads a bounded child mapping +inventory. Every file-backed mapping must match the device/inode of a hash-checked +native library or one of the two sealed descriptors. All required libraries and +both descriptors must be present. Anonymous/kernel vDSO/vvar mappings have no file +identity. The resulting map identities are retained as observations, not portable +file hashes. The captured policy supplies the corresponding content identities. + +This is qualified for the reviewed native binary without dynamic plugins. +Trusted installation files and `/etc/ld.so.preload` absence must remain immutable +for the attempt; hostile administrators are outside the contract. The wrapper +only cleans up its immediate child on failure. Existing helper subreaping, +pidfds, bounded cancellation, and unknown fences own descendant cessation. + +## Wire and lossless data + +Native READY is one little-endian u32-length JSON envelope. Each `02` byte requests +one accepted frame: another JSON envelope followed by raw RHS and pressure f64, +three MAC velocity f32 arrays, then tracer f32. Final `00` requests stop. There +must be exactly the requested frames and no trailing bytes. READY controls, frame +indices/times, finite field values, bounded tracer, and native divergence/Courant +diagnostics are checked. Wire is limited to 4 MiB, native diagnostics to 64 KiB, +JSON messages to 64 KiB, and the wrapper's protocol observation to 100 seconds. + +Array storage is X-fastest; NPY uses C-order `(z,y,x)`. RHS/pressure/tracer are +cell-centered. Velocity X/Y/Z respectively adds one sample to the X/Y/Z axis. +Wrapping adds only NPY headers; original element bytes, including signed zero and +f64 precision, are unchanged. RHS is preprojection, pressure solved, velocity +corrected and tracer advected. Native length/time/field units remain unspecified. +No simultaneous-stage interpretation is implied. + +`fields.zip` contains a generic `scientific_field_sequence_v1` manifest, one +uncompressed NPY per field/frame, and exact original wire, build receipt and +producer receipt plus the exact captured native policy. The declared family binds scene/initial-condition convention, +grid and seed; frame count and build hash cannot create an independent family. +The generic parser also applies its canonical source/state protection. + +Outputs are `fields.zip` (role `scientific_fields`), `wire.bin` (`native_wire`), +`producer-receipt.json` (`producer_receipt`), and `native-diagnostics.txt` +(`native_diagnostics`). Receipt fields bind controls, binary/build/policy/wrapper +hashes, original command bytes, wire bytes/hash, diagnostic hash, loaded mapping +observations, READY and frame reports, and qualification limits. An artifact's +role does not establish format validity or authorize import/review. + +## Verification and limits + +Focused tests include authored malformed-wire/format fixtures, immutable input +handling, recipe/project setup, family protection, and actual sealed-loader +32³/two-frame runs for seeds 100 and 101 when the pinned optional build exists. +Missing native prerequisites are explicit skips, never mocked acceptance. +Independent end-to-end qualification owns approved Runs completion, NumPy/native +cold checks, import, review, freeze, original download, restart, and cancellation. +This generation checkpoint does not qualify training, Pumas/GPU execution, +historical benchmark reproduction, or general scientific accuracy. diff --git a/docs/contracts/scientific-fields.md b/docs/contracts/scientific-fields.md index 37dfc499..eed74272 100644 --- a/docs/contracts/scientific-fields.md +++ b/docs/contracts/scientific-fields.md @@ -73,6 +73,32 @@ files remain unchanged. `asset(record)` verifies and returns captured bytes plus verifies complete ZIP identity. Symlinks, nonregular files, oversize sources, and changed captures are unavailable. +Completed external Run artifacts have a separate source method: +`admit_run_artifact(run_id, name, expected_sha256, body)`. The service requires +edit and inspect permissions, calls `Runs.capture_artifact` itself, and accepts no +caller-supplied provenance. HTTP exposes +`POST /api/workspace/runs/:id/import-data/:name` with the expected hash and the +same optional import metadata. Output role `scientific_fields` enables the UI +action; the actual ZIP must still pass the full scientific parser. + +The `run_artifact_import` provenance contains `run_source`, never `project_source`. +Its closed schema retains version, catalog/Run/attempt identities, request and +material hashes, artifact name/role/size/hash, custody identity, custody receipt +hash and canonical base64 of the original receipt (at most 16 KiB). Run/attempt, +request and receipt identities must agree; custody must be completed and drained. +Artifact size and hash must match the captured bundle. Frozen readers reuse these +checks without opening the producer, live runtime or original execution paths. +This is internally consistent retained execution evidence, not independent +proof of historical database approvals or scientific correctness. + +Import creates a draft. Review and freeze remain separate permissioned operations. +Cancelled, unknown, incomplete or changed results cannot import. Duplicate content +returns a conflict and creates no second record. After a lost acknowledgement, +inspect the existing catalog record by its content hash; do not rerun production. +Rollback removes unpublished bytes; uncertain commit retains bytes and catalog +state for reconciliation. Original `project_data_import` representations remain +unchanged, and mixed provenance variants are rejected. + `inspect(id, frame, field, axis, index)` returns a complete bounded orthogonal slice (maximum 129×129 scalars), original time/stage/units/dtype/centering, shape ZYX, and grid geometry. Z slices have Y rows/X columns, Y slices Z rows/X columns, diff --git a/docs/plans/generic-scientific-projects/execution-ledger.md b/docs/plans/generic-scientific-projects/execution-ledger.md index c99105a5..1a7047b0 100644 --- a/docs/plans/generic-scientific-projects/execution-ledger.md +++ b/docs/plans/generic-scientific-projects/execution-ledger.md @@ -85,3 +85,23 @@ Implemented external graph/preflight, exact actor-bound approvals/schema8, captu Full stable-source suite749 passes856.353s with2 optional-reader skips separately covered20 tests; all176 Python sources/tests unchanged. Focused approvals/storage50, host/retained28, helper11 and composed external15 pass. Existing31 JS checks and new external controller pass. Actual owner/agent desktop/narrow browser, cancellation/restart and exact downloads pass; project/recovery regressions pass. Native237-byte READY/STOP reaches exit0/ECHILD only, no simulation/model claim. Independent review findings fixed with regressions; no remaining local blocker. See [P3b verification](reports/p3b-verification.md). A further disconnect notice did not block tools or remove work; branch and test progress were checked directly. Fresh main remainsc522bf6; PR32/33 and38–47 preserved, all unmerged. Commit/push/draft publication follows; exact-head hosted lifecycle/UI qualify P4 admission. No merge or deployment. + + +## 2026-10-13 — P3b hosted qualification and P4 admission + +PR48 exact7d59060 passed hosted UI38440828965 and lifecycle38440828870 (132 steps). All work is pushed; no merge or deployment. Fresh main remainsc522bf6. Created feature/native-scientific-generation-20261013. Independent review accepted bounded native closure and server-derived Run artifact provenance, preserving old Python-only material identities and project-source release bytes. P4 exact API/write ownership and acceptance are recorded in [admission](reports/p4-admission.md). + + +## 2026-10-13 — P4 implementation and local qualification + +Implemented optional native closure/loader policy, the bounded Rheon adapter, +server-derived Run artifact imports and compact Data navigation. Independent +source review found no blocking issue. Full Python suite771 passes962.842s; +two optional skips are separately covered20reader tests. All185runtime/unit-test +sources stayed fixed; one standalone oracle refinement was independently used by +the final real browser. Focused20host/helper,9adapter,6import/HTTP,9reader/restore, +11retainedRun tests and affectedcontrollers pass. Actualnative browser6/6 gates +passes seed100/101, allarrays/slices, review/freeze, cancellation and restart; +no-adapter actualstartup also passes. Restore qualification exposed/fixed retained +absolute-path validation without rewriting receipt hashes. See P4verification. +Commit/push/draft publication follows; exact-head hosted CI gates P5. No merge. diff --git a/docs/plans/generic-scientific-projects/plan.md b/docs/plans/generic-scientific-projects/plan.md index 79212004..581843e2 100644 --- a/docs/plans/generic-scientific-projects/plan.md +++ b/docs/plans/generic-scientific-projects/plan.md @@ -2,9 +2,9 @@ **Plan status:** `Active` -**Current phase:** P3a qualified at published commit d3c0e09; P3b approved external recipes admitted on feature/approved-external-recipes-20261013. +**Current phase:** P3 qualified at published commit 7d59060; P4 bounded native generation admitted on feature/native-scientific-generation-20261013. -**Next slice:** P3b — exact material approval, captured external recipes and descendant custody; see the P3b admission report for the reviewed contract and write set. +**Next slice:** P4 — optional native generation and verified Run artifact import into Data; see [P4 admission](reports/p4-admission.md) for reviewed contracts and exact write boundaries. **Acceptance status:** `partial` @@ -34,7 +34,7 @@ The user approved publishing this plan and staged implementation on 2026-10-13. | --- | --- | --- | --- | --- | --- | --- | | G1 | Open/reopen a local project, inspect available/missing artifacts and runtime requirements, navigate all three workspaces without executing or modifying repository code | user-workflow | representative | automated | satisfied | [P1 observed workflow](reports/p1-verification.md) | | G2 | Scientific fields preserve exact float bytes, dimensions, staggered placement, units, time and trajectory identity through import, freeze, download and cold read | contract | representative | automated | satisfied | [Actual native and neutral custody workflows](reports/p2-verification.md) | -| G3 | An optional Rheon adapter exposes bounded generation controls and produces real generic scientific data; removing it leaves generic project/data use intact | integration | required-real | automated | pending | P4 actual native generator plus no-adapter startup | +| G3 | An optional Rheon adapter exposes bounded generation controls and produces real generic scientific data; removing it leaves generic project/data use intact | integration | required-real | automated | satisfied | [P4 actual generation and no-adapter startup](reports/p4-verification.md) | | G4 | Saved graph executes existing lab fitting and autonomous evaluation scripts; verified checkpoint, full-field comparisons and physics diagnostics survive restart with complete provenance | system | required-real | automated | pending | P5 lab verifier, numerical field comparison and new execution receipts | | G5 | Agent and UI clients use the same operations; stale requests, forged human review, unauthorized execution, path escape and approval reuse fail before side effects | contract | representative | automated | pending | P3/P6 public-interface denial and parity tests | | G6 | Admission is idempotent; cancel, crash and restart never duplicate work, publish partial success or silently leave descendants running | system | representative | automated | pending | P3 process-tree, storage-failure and lost-acknowledgement tests | @@ -200,7 +200,7 @@ Statuses below track the approved staged implementation. Before starting each mi **Acceptance gate:** G5 shared service portion and G6. Use a real bounded executable for lifecycle tests, clearly labelled non-scientific; actual native lab probe decides admission of P4. No false success after cancellation or partial outputs; stale authorization and forged review denied across direct HTTP and UI. -**Status:** `Verifying` +**Status:** `Accepted` ### P4: Optional Rheon generation controls @@ -212,7 +212,7 @@ Statuses below track the approved staged implementation. Before starting each mi **Acceptance gate:** G3 and P4 UI path. Real generated arrays pass cold generic and producer readers; invalid controls fail before spawn; removal of optional package leaves generic features operational. No fabricated native output or silent re-use of PR32's different fixed profile. -**Status:** `Planned` +**Status:** `Verifying` ### P5: Surrogate training and autonomous evaluation diff --git a/docs/plans/generic-scientific-projects/reports/p4-admission.md b/docs/plans/generic-scientific-projects/reports/p4-admission.md new file mode 100644 index 00000000..bca8b526 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p4-admission.md @@ -0,0 +1,140 @@ +# P4 admission — bounded native generation + +PR48 at `7d590605006f48f6c3f710b91a75ad4455d0a9d7` passed hosted UI +38440828965 and all 132 lifecycle steps in 38440828870. Fresh main remains +`c522bf6`. P4 uses `feature/native-scientific-generation-20261013`; no merge or +deployment is authorized. Apply the adopted architecture, Python/JavaScript, +untrusted-execution, persistence, dependency and verification standards. +Independent design review found this bounded scope viable at existing limits. + +## Goal and measured bounds + +Run optional native Rheon generation through the existing approved-recipe owner, +then import a verified artifact into Data for inspection, review, freeze and cold +read. Preserve Python-only recipes and historical scientific source identities. + +The pinned native binary is 766,832 bytes; the existing 32³/two-frame wire is +2,122,648 bytes. Its lossless generic bundle, including all twelve NPY arrays and +original evidence, fits the existing 40 MiB bundle and 64 MiB Run limits. Initial +recipe limits are 16 MiB outputs and 120 seconds. No global bounds increase. + +Source pins are Rheon `2209012bc5f902693d0f695ec1c2d854126af317` and lab +`e8a90bfef6b2955c502008ccc62443e29dd9f2ec`. Native binary SHA256 is +`4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff`. +P3b READY/STOP custody evidence alone does not qualify generation. + +## Optional adapter + +`adapters/rheon/generate.py` is a self-contained standard-library wrapper. Its +arguments are `--native`, `--build-receipt`, `--native-policy`, `--grid`, +`--frames`, `--seed` and `--output`. It owns native protocol parsing, bounded +controls, producer conversion and diagnostics; core owners have no new Rheon +branches. Initial controls are grid 8–32, frames 1–2 and a bounded integer seed, +with Jacobi fixed. Time step 1/60 and density 1.25 remain fixed. Native units are +explicitly unspecified. Preserve all six fields, exact float32/float64 payloads, +C-order ZYX/MAC placement, time and stage. + +Family identity binds the declared initial-condition convention, grid and seed; +requested frame count and build changes alone do not establish independent +trajectories. Binary/build hashes remain provenance. The exact original build +receipt is a genuine declared input; do not invent a dataset input for native +internal initial conditions. Outputs are `fields.zip`, `wire.bin`, a producer +receipt and bounded diagnostics. Role `scientific_fields` enables a generic UI +action but is not evidence that bytes are valid. + +`adapters/rheon/project.py --binary PATH --build-receipt PATH --destination PATH` +creates a new directory only. It copies exact input bytes and writes the generic +project/recipe manifests without building, executing or modifying source repos. + +## Generic native closure + +Optional `native_sources` declares 1–8 unique paths already present in `sources`. +An argument `{native_policy: source_path}` refers to generated captured policy +JSON under `native-policy/.json`. Material adds +`native_policies` keyed by source path only when native sources are declared. +Omitted-native definitions/material/specs gain no default fields. + +Apply a strict bounded ELF64 little-endian x86-64 policy recursively to the +executable, loader and every added DSO. Require the qualified PIE/interpreter +combination; reject malformed headers/string tables, path/token dependencies, +RPATH/RUNPATH, AUDIT/DEPAUDIT, FILTER and AUXILIARY mechanisms. Resolve the finite +closure only under trusted `/usr/lib/x86_64-linux-gnu`. Retain every dependency's +size/hash under existing runtime limits and ownership checks. + +Captured policy fields are `schema_version`, `policy`, `source {path, sha256}`, +`loader {path, bytes, sha256}`, `library_directory`, `loader_arguments`, +`runtime_files` and `absent_paths`. Policy is +`glibc_x86_64_no_preload_v1`. Loader arguments inhibit cache, fix the library +path and disable glibc hardware-capability substitutions. The only absent path +is `/etc/ld.so.preload`. + +Optional spec `native_loader_policy` names that fixed policy, not a general +filesystem-observation API. Existing launch/READY/GO verification checks trusted +`/etc` ownership and absence of any preload entry, including dangling symlinks. +The wrapper verifies/seals native and loader bytes and executes their descriptors, +then verifies actual mapped files at native READY before requesting frames. +This qualifies pinned trusted code under the existing immutable-installation +assumption, not arbitrary dynamic plugins or hostile administrators. No second +helper authority, scheduler or process owner. + +## Completed artifact to Data + +`Runs.capture_artifact(run_id, name, expected_sha256)` requires inspect authority, +verifies the complete published external result and custody, captures exact bytes, +rechecks their hash and returns a server-derived receipt. +`ScientificAssets.admit_run_artifact(run_id, name, expected_sha256, body)` requires +edit authority and calls that owner itself. HTTP exposes +`POST /api/workspace/runs/:id/import-data/:name`, accepting only the expected hash +and existing optional name/rights/groups/parents metadata. Parse strict bounded +JSON. Callers cannot supply provenance, review status or approvals. + +New provenance method `run_artifact_import` selects `run_source`, never +`project_source`. The closed source receipt has version, catalog/Run/attempt IDs, +request/material hashes, artifact name/role/size/measured hash, custody hash, +READY identity and canonical base64 of 1–16384 original custody receipt bytes. +Run/attempt/request identities must agree. Artifact size/hash must match captured +bytes. Shared pure `recipe_process.validate_receipt(raw, expected_identity)` +requires completed and drained custody; `load_receipt` delegates after reading. +Cold readers verify retained relationships, not independent historical database +approval authenticity or a recomputed full material document. + +Import creates a draft. Human review and freeze remain separate. Preserve the +old project representation and reject mixed variants. Existing duplicate-content +policy returns conflict without a second record; a lost acknowledgement is +reconciled by inspecting catalog content, not rerunning production. Preserve +rollback cleanup and uncertain-commit custody. No database schema change. + +Restore refinement: retain absolute execution paths/spec hashes as historical +evidence while reading artifacts beneath the restored catalog. Validate the full +spec against its canonical declared staging path. Dispatch refuses a path that +differs from the current owned staging location; never replay restored attempts. + +## Exact write ownership + +- Runtime: `recipe_contract.py`, `recipe_process.py`, corresponding two test files. +- Adapter: `adapters/rheon/generate.py`, `project.py`, `README.md`, + `docs/contracts/rheon-producer.md`, `tests/test_rheon_adapter.py`. +- Root: `runs.py`, `scientific_fields.py`, `app.py`, + `tests/test_run_scientific_import.py`, `tests/test_external_http.py` if needed. +- Reader: `scientific_release.py`, `tests/test_scientific_release.py`; an additional + scientific storage test file only if the existing restore fixture needs it. +- UI: `static/workspace-real.js`, `workspace-service.js`, `workspace-external.js`, + `workspaces.css`, `workspaces.js`, `tests/test_workspace_external_controller.cjs`. + The `workspaces.js` callback reuses generic catalog navigation and dirty/intent + guards instead of the classification correction path. +- Evidence: `tests/browser_workspace_native.cjs`, + `tests/qualify_native_generation.py`, dedicated reports/screenshots. +- Root documentation/packaging: README, external-recipes/scientific-fields + contracts, this plan directory and `.github/workflows/tests.yml`. + +## Acceptance + +Qualify actual 32³/two-frame seed 100 and a second seed through approved Runs; +compare every field against original wire with independent NumPy. Exercise exact +artifact import, Data inspection, owner review, freeze/download, cold read, +restart and relocated complete backup restore. Check frame-count family stability, +invalid controls before execution, cancellation without partial import, artifact +drift, rollback/unknown acknowledgement, authority denials, and desktop/narrow/ +keyboard UI. Preserve generic startup without the optional adapter and old +Python/project-source behavior. Native generation is not P5 training, GPU, +performance or general scientific-quality acceptance. diff --git a/docs/plans/generic-scientific-projects/reports/p4-verification.md b/docs/plans/generic-scientific-projects/reports/p4-verification.md new file mode 100644 index 00000000..3b50be7a --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p4-verification.md @@ -0,0 +1,93 @@ +# P4 qualification — optional native generation + +Candidate branch: `feature/native-scientific-generation-20261013`, based on +qualified PR48 `7d59060`. This report records worktree evidence; published commit +and exact-head CI will be recorded after publication. No merge or deployment. + +## Implemented behavior + +Optional captured native sources add a recursively verified loader/library policy +to existing approved recipes. The optional Rheon wrapper generates bounded real +Jacobi trajectories and retains every native array without conversion. Completed +Run artifacts can enter Data through a server-derived source receipt; imports are +drafts and require separate human review/freeze. Existing Python-only recipe +shapes and project-source releases remain supported. + +A real complete backup/restore test exposed an existing relocation defect: +receipt-bound absolute execution paths were being compared with the new catalog +path. Validation now preserves the original declaration and complete spec hash; +artifact reads use restored owned paths, while dispatch rejects relocated paths. +No historical receipt or execution specification is rewritten. + +## Real native workflow + +The final browser run is `build/qa/workspace-native-generation/run-uxEq8B`. +Its [receipt](p4/browser-receipt.json) and six screenshots are retained beside +[candidate source hashes](p4/source-files.json). Chrome 151 and a loopback-only +application transport canary were used; this is not a recipe network sandbox. + +All six workflow gates passed: + +1. Owner-approved production Run executes the sealed native/loader path; invalid + controls fail preflight and changed artifacts cannot import. +2. Independent NumPy 2.5.4 decodes original native wire, verifies all twelve NPY + array payloads/shapes/dtypes/MAC locations and eighteen inspected planes, and + checks retained policy/mapped-library identities and bounded diagnostics. +3. Import creates a draft; separate human review and canonical freeze retain the + exact generated bundle and original custody receipt. +4. Real seed 101 changes arrays and family; seed 100 with one frame preserves + the trajectory family and first-frame bytes. +5. Cancellation reaches `queued → running → cancelling → cancelled` with drained + custody and no importable partial result. This observation does not assert + that native frame generation had begun before cancellation. +6. Restart preserves the completed Run, reviewed Data, exact frozen release and + unchanged project sources. + +The primary Run is `82e605bd024346d2ae89cf6cef1465b2`, 32³/two frames/seed 100. +Native binary SHA256 is +`4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff`. +Original build receipt SHA256 is +`332aa36d8ce02585be77fbd74f8a7dfcfe9a297a3054b5bb67461ee3556d1e40`. +The receipt records output and release hashes, runtime mappings, numerical +observations, second-seed checks and screenshots. Units remain explicitly native +and unspecified. This qualifies bounded generation/custody, not training, GPU +performance, general physics validity or P5 autonomous evaluation. + +## Focused checks + +- 20 host/helper tests pass, including recursive ELF rejection, captured native + policies, preload refusal and unchanged omitted-native shapes. +- 9 adapter tests pass with explicit qualified binary/receipt environment paths, + including real two-seed generation. Without those inputs, native-only tests + report explicit skips; authored fixtures are not real acceptance. +- 6 real Run import/HTTP tests pass: permission boundaries, draft-only admission, + original receipt bytes, duplicate refusal, changed bytes, noncompleted states, + rollback cleanup and unknown commit acknowledgement. +- 9 scientific reader tests pass with NumPy enabled: original project releases, + real neutral Run import/freeze/cold CLI, tampered/mixed provenance, and complete + PostgreSQL/artifact backup restoration to a different directory. Restored + `Runs.capture_artifact` returns the same bytes and source receipt. +- 11 retained external Runs tests pass after the relocation fix. The independent + point-cloud/volume reader environment passes all 20 checks without skips. +- External controller, workspace service, auth controller/helper and affected + JavaScript syntax checks pass. Existing recovery browser passes. +- Actual application startup with the optional adapters directory physically + absent passes owner authentication, static UI delivery, generic project + inspection, draft scientific import and exact Data slice inspection. No producer + executes and Runs stays empty. See [receipt](p4/no-adapter-receipt.json) and + [exact qualification procedure](p4/no-adapter-procedure.txt). +- Independent source review found no blocking issue. Plan-contract validation + and whitespace checks pass. No static type checker or Standards Engine + certification is claimed. + +Full Python regression passes: **771 tests in 962.842 seconds**, with two optional-reader skips separately covered by the 20-test independent reader run. All 185 runtime/unit-test Python sources remain unchanged. [Before/after inventories](p4/python-suite-source-files.json) record the separate standalone oracle refinement. Final published-head CI remains pending. The standalone +native NumPy oracle received an evidence-only validation refinement after the +full-suite inventory was taken; it is not imported by the unit suite and the +final real browser ran that revised oracle. Record both inventories explicitly. + +## Remaining scope + +P5 requires a separately reviewed captured CPU runtime, actual fitting/evaluation +and larger lossless result handling. P6/P7 remain planned. Dataset-workflows A8 +positive real-model proposals and original Rheon v2 packet retrieval/qualification +remain separate open gates. ASR and general 3D training are explicit extensions. diff --git a/docs/plans/generic-scientific-projects/reports/p4/browser-receipt.json b/docs/plans/generic-scientific-projects/reports/p4/browser-receipt.json new file mode 100644 index 00000000..64526cd9 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p4/browser-receipt.json @@ -0,0 +1,898 @@ +{ + "source_commit": "7d590605006f48f6c3f710b91a75ad4455d0a9d7", + "dirty": true, + "qualification": "Actual optional native generator through approved production Runs; exact field custody and bounded numerical observations", + "model_or_training_acceptance": false, + "general_physics_acceptance": false, + "gpu_or_performance_acceptance": false, + "checks": [ + "Actual native approved generation completes; invalid grid and stale/tampered artifact imports refuse before Data writes", + "All six native fields and eighteen inspected planes match independent NumPy/wire bytes, MAC placement and diagnostics", + "Separate explicit human review and canonical freeze preserve the exact generated bundle and verifiable run provenance", + "A second native seed changes actual arrays and family; shorter frame count preserves original initial-condition family and frame bytes", + "Cancellation publishes no importable partial scientific artifact", + "Server restart retains approved Run, reviewed data, exact frozen release and unchanged project sources" + ], + "captures": [ + { + "file": "native-pipeline-1440.jpg", + "width": 1440, + "height": 900, + "qualification": "Actual bounded native generation UI; no model/GPU acceptance" + }, + { + "file": "native-pipeline-390.jpg", + "width": 390, + "height": 900, + "qualification": "Actual bounded native generation UI; no model/GPU acceptance" + }, + { + "file": "native-runs-390.jpg", + "width": 390, + "height": 900, + "qualification": "Actual bounded native generation UI; no model/GPU acceptance" + }, + { + "file": "native-runs-1440.jpg", + "width": 1440, + "height": 900, + "qualification": "Actual bounded native generation UI; no model/GPU acceptance" + }, + { + "file": "native-data-1440.jpg", + "width": 1440, + "height": 900, + "qualification": "Actual bounded native generation and custody; no model/GPU acceptance" + }, + { + "file": "native-data-390.jpg", + "width": 390, + "height": 900, + "qualification": "Actual bounded native generation and custody; no model/GPU acceptance" + } + ], + "cancelled_run": { + "id": "5c7103d661af40c9a941e496f5c068ca", + "state": "cancelled", + "custody_phase": "drained", + "events": [ + { + "at": "2026-10-13T18:15:23.174325+00:00", + "message": "Exact owner-approved external recipe admitted.", + "state": "queued" + }, + { + "at": "2026-10-13T18:15:23.322932+00:00", + "message": "Starting the approved custodian; recipe waits behind its launch barrier.", + "state": "running" + }, + { + "at": "2026-10-13T18:15:23.375371+00:00", + "message": "Cancellation requested; waiting for confirmed process-group cessation.", + "state": "cancelling" + }, + { + "at": "2026-10-13T18:15:23.752463+00:00", + "message": "External process drained without successful publication.", + "state": "cancelled" + } + ] + }, + "run_id": "82e605bd024346d2ae89cf6cef1465b2", + "record_id": "80790a28f790430fb6cf4c6dc4373b3c", + "build_receipt_sha256": "332aa36d8ce02585be77fbd74f8a7dfcfe9a297a3054b5bb67461ee3556d1e40", + "binary_sha256": "4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff", + "cold": { + "scope": "Actual native generation custody and bounded full-grid numerical observations", + "numpy": "2.5.4", + "bundle_sha256": "9ab28ba4dec22448478a136dcae634e0dcdc43c36866ef4d091770c5f4f68a2c", + "wire_sha256": "6b16244014dca96f81c2101149aa98e9692bb84d8b4de0671dbb13831ed1d4cb", + "grid": 32, + "frames": 2, + "seed": 100, + "family": { + "id": "f384aaff7ae7dd3481d063c6e8c2012bb878d32b839e4f4dd3abcfefd1a8c9e7", + "namespace": "rheon-trajectory-probe-scene-v1" + }, + "arrays": [ + { + "frame": 0, + "field": "rhs", + "dtype": " 32: raise WorkbenchError('Declare at most 32 typed parameters.') for key, rule in value['parameters'].items(): @@ -109,6 +113,8 @@ def decode_definition(raw): if not isinstance(arg, str) or arg not in value['inputs']: raise WorkbenchError('Unknown input argument.') elif kind == 'source': if not isinstance(arg, str) or arg not in paths: raise WorkbenchError('Unknown source argument.') + elif kind == 'native_policy': + if not isinstance(arg,str) or arg not in value.get('native_sources',[]): raise WorkbenchError('Native policy argument must select a declared native source.') elif kind != 'output_dir' or arg is not True: raise WorkbenchError('Unsupported argument token.') outputs = value['outputs'] @@ -233,6 +239,151 @@ def runtime_identity(profile=None): 'assumption': TRUST, 'flags': ['-I', '-S', '-B']} +NATIVE_POLICY = 'glibc_x86_64_no_preload_v1' +NATIVE_LIBRARY = '/usr/lib/x86_64-linux-gnu' +NATIVE_LOADER = NATIVE_LIBRARY + '/ld-linux-x86-64.so.2' +NATIVE_ARGUMENTS = ['--inhibit-cache', '--library-path', NATIVE_LIBRARY, '--glibc-hwcaps-mask', ''] + + +def native_policy_path(source): + return 'native-policy/' + hashlib.sha256(source.encode()).hexdigest() + '.json' + + +def strict_native_elf(raw, *, executable=False): + """Closed ELF64/x86-64 dynamic dependency policy, without loader execution.""" + def fail(): + raise WorkbenchError('Native ELF is malformed or outside the qualified loader policy.', 'unsupported', 400) + if type(raw) is not bytes or len(raw) < 64 or raw[:7] != b'\x7fELF\x02\x01\x01': fail() + etype, machine, version = struct.unpack_from(' len(raw)): fail() + segments = [struct.unpack_from(' len(raw) or (kind == 1 and length > memory): fail() + if kind == 1: loads.append((pos,va,length)) + if kind == 3: + if interpreter is not None or not 1 <= length <= 256: fail() + value = raw[pos:pos+length] + if value[-1:] != b'\0' or b'\0' in value[:-1]: fail() + try: interpreter = value[:-1].decode('ascii') + except UnicodeError: fail() + if interpreter not in ('/lib64/ld-linux-x86-64.so.2', NATIVE_LOADER): fail() + if kind == 2: + if dynamic_seen or length % 16 or not 16 <= length <= 65536: fail() + dynamic_seen = True + terminated = False + for at in range(pos,pos+length,16): + tag, value = struct.unpack_from('= sizes[0]: fail() + end = raw.find(b'\0',base+value,base+sizes[0]) + if end < 0: fail() + try: name = raw[base+value:end].decode('ascii') + except UnicodeError: fail() + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}',name) or name in needed: fail() + needed.append(name) + if len(needed) > 64: fail() + return needed + + +def native_policies(definition, sources, runtime): + """Capture every recursively required trusted native library into material.""" + import recipe_process + recipe_process.verify_native_loader_policy(runtime['trusted_install_owner_uid']) + result = {} + for source in definition['native_sources']: + needed = strict_native_elf(sources[source], executable=True) + pending = [Path(NATIVE_LOADER), *(Path(NATIVE_LIBRARY)/name for name in needed)] + files = {} + while pending: + path = pending.pop().resolve(strict=True) + if str(path) in files: continue + if path.parent != Path(NATIVE_LIBRARY) or len(files) >= 64: + raise WorkbenchError('Native library closure escapes its finite trusted directory.') + for ancestor in (path,*path.parents): + if ancestor == Path('/'): break + info = ancestor.stat() + if info.st_uid != runtime['trusted_install_owner_uid'] or info.st_mode & 0o022: + raise WorkbenchError('Native runtime owner or permissions differ from trusted installation.') + if ancestor == Path('/usr'): break + info = path.stat() + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_SOURCE_BYTES: + raise WorkbenchError('Native library exceeds its regular-file bound.') + raw = path.read_bytes(); after = path.stat() + if (info.st_ino,info.st_size,info.st_mtime_ns,info.st_ctime_ns) != (after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns): + raise WorkbenchError('Native runtime changed during preflight.', 'stale',409) + dependencies = strict_native_elf(raw) + files[str(path)] = {'bytes':len(raw),'sha256':file_digest(raw)} + pending.extend(Path(NATIVE_LIBRARY)/name for name in dependencies) + policy = {'schema_version':1,'policy':NATIVE_POLICY,'source':{'path':source,'sha256':file_digest(sources[source])}, + 'loader':dict(path=NATIVE_LOADER,**files[NATIVE_LOADER]),'library_directory':NATIVE_LIBRARY, + 'loader_arguments':list(NATIVE_ARGUMENTS), + 'runtime_files':[dict(path=path,**record) for path,record in sorted(files.items())], + 'absent_paths':['/etc/ld.so.preload']} + result[source] = policy + runtime['files'].update(files) + runtime['files'] = dict(sorted(runtime['files'].items())) + runtime['closure_sha256'] = digest(runtime['files']) + return result + + +def validate_native_policies(material, definition): + """Validate retained policy declarations with no live runtime dependency.""" + policies = material.get('native_policies') + if 'native_sources' not in definition: + if 'native_policies' in material: raise WorkbenchError('Unexpected native policy material.') + return {} + if not isinstance(policies,dict) or set(policies) != set(definition['native_sources']): + raise WorkbenchError('Native source and policy inventories differ.') + runtime = material.get('runtime') + if not isinstance(runtime,dict) or not isinstance(runtime.get('files'),dict): + raise WorkbenchError('Invalid native runtime closure.') + sources = {item['path']:item['sha256'] for item in definition['sources']} + generated = {} + for source,policy in policies.items(): + closed(policy, ('schema_version','policy','source','loader','library_directory','loader_arguments','runtime_files','absent_paths')) + if (type(policy['schema_version']) is not int or policy['schema_version'] != 1 or policy['policy'] != NATIVE_POLICY + or policy['source'] != {'path':source,'sha256':sources[source]} + or policy['library_directory'] != NATIVE_LIBRARY or policy['loader_arguments'] != NATIVE_ARGUMENTS + or policy['absent_paths'] != ['/etc/ld.so.preload']): + raise WorkbenchError('Native loader policy is inconsistent.') + rows = policy['runtime_files'] + if not isinstance(rows,list) or not 1 <= len(rows) <= 64: raise WorkbenchError('Invalid native runtime inventory.') + seen = set() + for row in rows: + closed(row, ('path','bytes','sha256')) + path = row['path'] + if not isinstance(path,str) or str(Path(path).parent) != NATIVE_LIBRARY or path in seen: + raise WorkbenchError('Native library path differs from policy.') + relative_path(path[1:]); seen.add(path) + expected = material['runtime']['files'].get(path) + if not isinstance(expected,dict) or set(expected) != {'bytes','sha256'} or encode(row) != encode(dict(path=path,**expected)): + raise WorkbenchError('Native library differs from retained runtime closure.') + loader = policy['loader'] + if NATIVE_LOADER not in seen or loader != dict(path=NATIVE_LOADER,**material['runtime']['files'][NATIVE_LOADER]): + raise WorkbenchError('Native loader identity is inconsistent.') + generated[native_policy_path(source)] = encode(policy).encode() + return generated + + def graph_parameters(graph): nodes = graph['nodes'] if len(nodes) != 3 or {n['type'] for n in nodes} != set(TYPES) or any(n['definition_version'] != 1 for n in nodes): @@ -281,12 +432,24 @@ def prepare(projects, graph, execution_sha256, profile=None): runtime = runtime_identity(profile) except (OSError, struct.error, UnicodeError): raise WorkbenchError('Qualified system runtime cannot be safely inspected.', 'unavailable', 409) from None + policies = None + if 'native_sources' in definition: + try: + policies = native_policies(definition,sources,runtime) + except OSError: + raise WorkbenchError('Native runtime cannot be safely inspected.', 'unavailable',409) from None + for source,policy in policies.items(): + captured[native_policy_path(source)] = encode(policy).encode() + if len(captured) > MAX_FILES or sum(map(len,captured.values())) > MAX_CAPTURE_BYTES: + raise WorkbenchError('Generated native policies exceed the capture bound.') host = host_identity() material = {'schema_version': 1, 'catalog_id': projects.db.dataset_id, 'project': inspected['receipt'], 'graph_execution_sha256': execution_sha256, 'recipe': definition, 'parameters': values, 'inputs': inputs, 'captures': {path: {'bytes':len(raw),'sha256':file_digest(raw)} for path,raw in sorted(captured.items())}, 'runtime': runtime, 'host_files': host, 'permissions': {'filesystem':'trusted-local', 'network':'not_isolated'}, 'environment': {'LANG':'C.UTF-8'}, 'resources': definition['resources']} + if policies is not None: material['native_policies'] = policies + validate_material(material) return {'valid':True, 'status':'valid', 'material':material, 'material_sha256':digest(material), 'validation_token':digest(material), 'parameter_definitions':definition['parameters'], 'outputs':definition['outputs'], 'runtime':runtime, 'graph_revision':graph['revision'], 'execution_sha256':execution_sha256}, captured @@ -295,7 +458,7 @@ def prepare(projects, graph, execution_sha256, profile=None): def validate_material(material): """Cold structural custody checks; no filesystem access or current-code comparison.""" closed(material, ('schema_version', 'catalog_id', 'project', 'graph_execution_sha256', 'recipe', - 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources')) + 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources'), ('native_policies',)) if type(material['schema_version']) is not int or material['schema_version'] != 1 or not isinstance(material['catalog_id'], str) or not material['catalog_id']: raise WorkbenchError('Invalid execution material identity.') def sha(value): @@ -318,14 +481,17 @@ def receipt(value): if (item['project_id'], item['manifest_sha256']) != (material['project']['project_id'], material['project']['manifest_sha256']): raise WorkbenchError('Input receipt belongs to another project snapshot.') expected['input/'+name] = item['measured_sha256'] + generated = validate_native_policies(material,definition) + expected.update({path:file_digest(raw) for path,raw in generated.items()}) captures = material['captures'] if not isinstance(captures, dict) or set(captures) != set(expected) or len(captures) > MAX_FILES: raise WorkbenchError('Captured inventory differs from the declared source and inputs.') total = 0 for path, item in captures.items(): relative_path(path); closed(item, ('bytes','sha256')); sha(item['sha256']) - limit = MAX_SOURCE_BYTES if path.startswith('source/') else definition['inputs'][path.split('/',1)[1]]['max_bytes'] + limit = MAX_SOURCE_BYTES if path.startswith(('source/','native-policy/')) else definition['inputs'][path.split('/',1)[1]]['max_bytes'] bounded_int(item['bytes'],0,limit,'Capture bytes'); total += item['bytes'] + if path in generated and item['bytes'] != len(generated[path]): raise WorkbenchError('Generated native policy size differs.') if item['sha256'] != expected[path]: raise WorkbenchError('Captured digest contradicts source/input identity.') if total > MAX_CAPTURE_BYTES: raise WorkbenchError('Captured material exceeds byte bound.') runtime = material['runtime'] @@ -363,13 +529,16 @@ def spec_for_material(material, staging): item = material['parameters'][value]; arg = ('true' if item else 'false') if type(item) is bool else str(item) elif kind == 'input': arg = str(staging/'input'/value) elif kind == 'source': arg = str(staging/'source'/value) + elif kind == 'native_policy': arg = str(staging/native_policy_path(value)) else: arg = str(output) argv.append(arg) - return {'schema_version':1,'argv':argv,'env':dict(material['environment']),'cwd':str(staging), 'stdin_base64':base64.b64encode(b'').decode(), + spec = {'schema_version':1,'argv':argv,'env':dict(material['environment']),'cwd':str(staging), 'stdin_base64':base64.b64encode(b'').decode(), 'wall_seconds':definition['resources']['wall_seconds'], 'executable_sha256':runtime['executable_sha256'], 'max_output_bytes':definition['resources']['max_output_bytes'], 'output_dir':str(output),'captured_files':records, 'runtime_files':[dict(path=path,**record) for path,record in sorted(runtime['files'].items())], 'trusted_install_owner_uid':runtime['trusted_install_owner_uid']} + if 'native_policies' in material: spec['native_loader_policy'] = NATIVE_POLICY + return spec def validate_spec(material, spec, staging): diff --git a/recipe_process.py b/recipe_process.py index 0adc669b..96aeb92a 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -72,13 +72,15 @@ def absolute(value): def validate_spec(value): keys = {'schema_version','attempt_id','request_sha256','argv','env','cwd','stdin_base64', 'wall_seconds','executable_sha256','max_output_bytes','output_dir','captured_files','runtime_files','trusted_install_owner_uid'} - if type(value) is not dict or set(value) != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: + if type(value) is not dict or set(value) - {'native_loader_policy'} != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: raise ProcessError('Unsupported custody spec.') if not isinstance(value['attempt_id'], str) or not ID.fullmatch(value['attempt_id']): raise ProcessError('Invalid attempt identity.') for key in ('request_sha256','executable_sha256'): if type(value[key]) is not str or not HASH.fullmatch(value[key]): raise ProcessError('Invalid custody hash.') + if 'native_loader_policy' in value and value['native_loader_policy'] != 'glibc_x86_64_no_preload_v1': + raise ProcessError('Unsupported native loader policy.') argv, env = value['argv'], value['env'] if type(argv) is not list or not 1 <= len(argv) <= 64 or not all(text(arg) for arg in argv) or not absolute(argv[0]): raise ProcessError('Invalid captured argument vector.') @@ -134,7 +136,25 @@ def read_regular(path, maximum): return raw +def verify_native_loader_policy(owner_uid): + """The one admitted native policy permits no global loader preload entry.""" + descriptor = os.open('/etc', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + info = os.fstat(descriptor) + if info.st_uid != owner_uid or info.st_mode & 0o022: + raise ProcessError('Native loader configuration directory is not trusted.', 'unavailable') + try: + os.stat('ld.so.preload', dir_fd=descriptor, follow_symlinks=False) + except FileNotFoundError: + return + raise ProcessError('Native loader requires absent /etc/ld.so.preload.', 'unavailable') + finally: + os.close(descriptor) + + def verify_captures(spec): + if 'native_loader_policy' in spec: + verify_native_loader_policy(spec['trusted_install_owner_uid']) for item in spec['captured_files']+spec['runtime_files']: raw = read_regular(item['path'], item['bytes']) if len(raw) != item['bytes'] or digest(raw) != item['sha256']: @@ -310,9 +330,15 @@ def launch(spec, custody_dir): def load_receipt(path, expected_identity): + return validate_receipt(read_regular(path, 16384), expected_identity) + + +def validate_receipt(raw, expected_identity): + """Validate retained exact receipt bytes without requiring a live filesystem.""" + if type(raw) is not bytes or not 1 <= len(raw) <= 16384: + raise ProcessError('Custody receipt must contain 1–16384 bytes.') if type(expected_identity) is not dict or set(expected_identity) != {'attempt_id','request_sha256','spec_sha256','helper'} or not valid_identity(expected_identity['helper']): raise ProcessError('An exact READY identity is required to read a receipt.') - raw = read_regular(path,16384) value = decode(raw) fields = {'schema_version','attempt_id','request_sha256','spec_sha256','helper','go_received','spawn_closed', 'drained','echild','outcome','reason','leader_returncode','streams'} diff --git a/runs.py b/runs.py index 72976f08..8d59f1b0 100644 --- a/runs.py +++ b/runs.py @@ -1,5 +1,6 @@ """Durable admission, one local worker, and verified immutable result custody.""" from access import authorized +import base64 import copy import hashlib import json @@ -107,7 +108,15 @@ def _load(self, run_id): if self._external(run): import recipe_contract import recipe_process - recipe_contract.validate_spec(run['material'], run['spec'], self._external_directory(run)) + # Absolute execution paths are historical evidence, not restore + # destinations. Never rewrite a spec bound by a custody receipt. + declared = run['spec']['cwd'] + if (type(declared) is not str or not Path(declared).is_absolute() + or '..' in Path(declared).parts or str(Path(declared)) != declared + or Path(declared).name != '.external-' + run['id'] + or Path(declared).parent.name != 'workspace-runs'): + raise ValueError('Invalid retained external staging location.') + recipe_contract.validate_spec(run['material'], run['spec'], declared) if (run['graph'].get('recipe') != 'external_recipe' or run['material']['catalog_id'] != self.db.dataset_id or digest(run['material']) != run['material_sha256'] @@ -452,6 +461,8 @@ def _execute_external(self, run_id): self.access.require_job('runs', run_id, run['admission_id'], run['request_sha256']) if run['state'] != 'queued' or self._external_blocked: return + if run['spec']['cwd'] != str(self._external_directory(run)): + raise WorkbenchError('Restored execution paths cannot be dispatched. Approve a new attempt.', 'stale', 409) self._event(run, 'running', 'Starting the approved custodian; recipe waits behind its launch barrier.') run['started_at'] = timestamp() self._save(run) @@ -1179,6 +1190,43 @@ def _verified_artifacts(self, run): raise WorkbenchError('Published artifacts differ from the admitted result receipt.', 'unavailable', 409) return artifacts + @authorized('inspect', 'runs.capture_artifact') + def capture_artifact(self, run_id, name, expected_sha256): + """Capture a completed external artifact with server-derived custody evidence.""" + import recipe_process + if type(expected_sha256) is not str or not re.fullmatch('[a-f0-9]{64}', expected_sha256): + raise WorkbenchError('Supply the expected artifact SHA256.') + with self.lock: + run = self._load(run_id) + if not self._external(run): + raise WorkbenchError('Scientific import requires a completed external Run.', 'unsupported', 400) + item = next((item for item in self._verified_artifacts(run) if item['name'] == name), None) + if item is None: + raise WorkbenchError('Unknown external artifact.', 'unavailable', 404) + if item['sha256'] != expected_sha256: + raise WorkbenchError('Artifact differs from the expected SHA256.', 'conflict', 409) + raw = self._read_regular(self.path / run_id / name, item['bytes']) + if len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != expected_sha256: + raise WorkbenchError('Artifact changed during capture.', 'conflict', 409) + receipt_raw = self._read_regular(self.path / run_id / 'custody-receipt.json', 16384) + if hashlib.sha256(receipt_raw).hexdigest() != run['custody_receipt_sha256']: + raise WorkbenchError('Custody receipt changed during capture.', 'conflict', 409) + try: + receipt = recipe_process.validate_receipt(receipt_raw, run['custody_identity']) + except recipe_process.ProcessError as error: + raise WorkbenchError('Retained custody receipt cannot be verified.', 'unavailable', 409) from error + if not receipt['drained'] or receipt['outcome'] != 'completed': + raise WorkbenchError('Run custody did not complete.', 'unavailable', 409) + source = {'schema_version': 1, 'catalog_id': run['dataset_id'], + 'run_id': run['id'], 'attempt_id': run['spec']['attempt_id'], + 'request_sha256': run['request_sha256'], 'material_sha256': run['material_sha256'], + 'artifact_name': item['name'], 'artifact_role': item['role'], + 'artifact_bytes': item['bytes'], 'measured_sha256': item['sha256'], + 'custody_receipt_sha256': run['custody_receipt_sha256'], + 'custody_identity': copy.deepcopy(run['custody_identity']), + 'custody_receipt_base64': base64.b64encode(receipt_raw).decode('ascii')} + return raw, source + @authorized('inspect', 'runs.artifact') def artifact(self, run_id, name): with self.lock: diff --git a/scientific_fields.py b/scientific_fields.py index b43319e9..73f8c342 100644 --- a/scientific_fields.py +++ b/scientific_fields.py @@ -1,6 +1,8 @@ """Lossless bounded structured-grid fields; no producer execution or physics policy.""" from access import authorized import ast +import base64 +import binascii from decimal import Decimal, InvalidOperation import hashlib import io @@ -272,6 +274,74 @@ def parse_bundle(raw): invalid('Malformed scientific bundle: ' + type(error).__name__) +def validate_source(method, source, raw): + """Validate retained source declarations against captured scientific bytes. + + Run evidence establishes internal custody relationships. It does not prove + historical database authority or reproduce the full approved material. + """ + digest = hashlib.sha256(raw).hexdigest() + if method == 'project_data_import': + exact(source, ('project_id', 'manifest_sha256', 'reference_id', 'path', 'measured_sha256'), 'Project source') + if type(source['project_id']) is not str or not re.fullmatch('[a-f0-9]{32}', source['project_id']): + invalid('Invalid project source ID.') + for key in ('manifest_sha256', 'measured_sha256'): + if type(source[key]) is not str or not HASH.fullmatch(source[key]): + invalid('Invalid project source hash.') + text(source['reference_id'], 64, 'Project reference') + text(source['path'], 512, 'Project source path') + elif method == 'run_artifact_import': + import recipe_process + exact(source, ('schema_version', 'catalog_id', 'run_id', 'attempt_id', + 'request_sha256', 'material_sha256', 'artifact_name', + 'artifact_role', 'artifact_bytes', 'measured_sha256', + 'custody_receipt_sha256', 'custody_identity', + 'custody_receipt_base64'), 'Run source') + if type(source['schema_version']) is not int or source['schema_version'] != 1: + invalid('Unsupported Run source version.') + for key in ('catalog_id', 'run_id', 'attempt_id'): + if type(source[key]) is not str or not re.fullmatch('[a-f0-9]{32}', source[key]): + invalid('Invalid Run source identity.') + for key in ('request_sha256', 'material_sha256', 'measured_sha256', 'custody_receipt_sha256'): + if type(source[key]) is not str or not HASH.fullmatch(source[key]): + invalid('Invalid Run source hash.') + if (type(source['artifact_name']) is not str + or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,119}', source['artifact_name']) + or source['artifact_name'] in ('artifact-manifest.json', 'custody-receipt.json') + or type(source['artifact_role']) is not str or not TOKEN.fullmatch(source['artifact_role'])): + invalid('Invalid Run artifact declaration.') + if type(source['artifact_bytes']) is not int or source['artifact_bytes'] != len(raw): + invalid('Run artifact size differs from captured bytes.') + identity = source['custody_identity'] + exact(identity, ('attempt_id', 'request_sha256', 'spec_sha256', 'helper'), 'Custody identity') + if (source['run_id'] != source['attempt_id'] or source['attempt_id'] != identity['attempt_id'] + or source['request_sha256'] != identity['request_sha256'] + or type(identity['spec_sha256']) is not str or not HASH.fullmatch(identity['spec_sha256'])): + invalid('Run source and custody identities differ.') + encoded = source['custody_receipt_base64'] + if type(encoded) is not str or not 1 <= len(encoded) <= 21848: + invalid('Invalid retained custody receipt encoding.') + try: + receipt_raw = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as error: + raise WorkbenchError('Scientific fields: Invalid retained custody receipt encoding.') from error + if (not 0 < len(receipt_raw) <= 16384 + or base64.b64encode(receipt_raw).decode('ascii') != encoded + or hashlib.sha256(receipt_raw).hexdigest() != source['custody_receipt_sha256']): + invalid('Retained custody receipt bytes differ.') + try: + receipt = recipe_process.validate_receipt(receipt_raw, identity) + except recipe_process.ProcessError as error: + raise WorkbenchError('Scientific fields: Invalid retained custody evidence.') from error + if receipt['outcome'] != 'completed' or not receipt['drained']: + invalid('Run source requires completed and drained custody.') + else: + invalid('Unsupported scientific source method.') + if source['measured_sha256'] != digest: + invalid('Captured bytes differ from their source receipt.', 'conflict', 409) + return dict(source) + + class ScientificAssets: def __init__(self, workbench): self.access = workbench.access @@ -280,6 +350,14 @@ def __init__(self, workbench): @authorized('edit', 'scientific_fields.admit') def admit(self, raw, body, source): + return self._admit(raw, body, 'project_data_import', source) + + @authorized('edit', 'scientific_fields.admit_run_artifact') + def admit_run_artifact(self, run_id, name, expected_sha256, body): + raw, source = self.workbench.dataset.runs.capture_artifact(run_id, name, expected_sha256) + return self._admit(raw, body, 'run_artifact_import', source) + + def _admit(self, raw, body, method, source): from immutable_assets import publish_file if type(body) is not dict or set(body) - {'name','rights','groups','parents'}: invalid('Unexpected import metadata fields.') @@ -289,19 +367,11 @@ def admit(self, raw, body, source): groups = strings(body.get('groups', []), 'Protected groups', maximum=27) parents = strings(body.get('parents', []), 'Parent IDs') groups = list(dict.fromkeys(groups + meta['protected_groups'])) - exact(source, ('project_id','manifest_sha256','reference_id','path','measured_sha256'), 'Project source') - if type(source['project_id']) is not str or not re.fullmatch('[a-f0-9]{32}', source['project_id']): - invalid('Invalid project source ID.') - for key in ('manifest_sha256','measured_sha256'): - if type(source[key]) is not str or not HASH.fullmatch(source[key]): - invalid('Invalid project source hash.') - text(source['reference_id'], 64, 'Project reference') - text(source['path'], 512, 'Project source path') + source = validate_source(method, source, raw) digest = hashlib.sha256(raw).hexdigest() - if source['measured_sha256'] != digest: - invalid('Captured project bytes differ from their source receipt.', 'conflict', 409) - origin = {'method':'project_data_import','rights':rights,'field_sequence':meta, - 'source_sha256':digest,'project_source':dict(source)} + source_key = 'project_source' if method == 'project_data_import' else 'run_source' + origin = {'method':method,'rights':rights,'field_sequence':meta, + 'source_sha256':digest,source_key:source} record_id, created = uuid.uuid4().hex, timestamp() w = self.workbench with w.lock, w.db.transaction(): diff --git a/scientific_release.py b/scientific_release.py index 2eed8358..47d6b828 100644 --- a/scientific_release.py +++ b/scientific_release.py @@ -7,7 +7,6 @@ import argparse import json from pathlib import Path -import re import sys import canonical_release @@ -55,20 +54,16 @@ def validate_row(row, release, used_ids): require(type(row['field_sequence']) is dict and encode(row['field_sequence']) == encode(metadata), 'field-sequence metadata differs from exact source.') provenance = row['provenance'] - require(provenance.get('method') == 'project_data_import' + require(provenance.get('method') in ('project_data_import', 'run_artifact_import') and type(provenance.get('field_sequence')) is dict and encode(provenance['field_sequence']) == encode(metadata) and provenance.get('source_sha256') == sha(raw), 'field-sequence provenance differs from exact source.') - source = provenance.get('project_source') - require(type(source) is dict - and set(source) == {'project_id', 'manifest_sha256', 'reference_id', 'path', 'measured_sha256'}, - 'invalid declared project source receipt.') - require(type(source['project_id']) is str and re.fullmatch('[a-f0-9]{32}', source['project_id']) - and type(source['manifest_sha256']) is str and re.fullmatch('[a-f0-9]{64}', source['manifest_sha256']) - and source['measured_sha256'] == sha(raw), 'declared project source identity differs from retained bytes.') - scientific_fields.text(source['reference_id'], 64, 'Project reference') - scientific_fields.text(source['path'], 512, 'Project source path') + method = provenance['method'] + source_key = 'project_source' if method == 'project_data_import' else 'run_source' + other_key = 'run_source' if method == 'project_data_import' else 'project_source' + require(other_key not in provenance, 'mixed project and Run source provenance is forbidden.') + scientific_fields.validate_source(method, provenance.get(source_key), raw) require(set(metadata['protected_groups']) <= set(row['groups']), 'protected field source, state or family is missing.') result = {key: row[key] for key in ('id', 'name', 'revision', 'source_revision', 'split', 'review')} diff --git a/static/workspace-external.js b/static/workspace-external.js index 5141fc9d..d36d946a 100644 --- a/static/workspace-external.js +++ b/static/workspace-external.js @@ -1,6 +1,6 @@ 'use strict'; // External graph projection only. The existing Pipeline and Runs owners admit all work. -window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,isDirty,replaceGraph,changeGraph,render,refresh,onRun}){ +window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,getRunId,isDirty,replaceGraph,changeGraph,render,refresh,onRun,onImported}){ const esc=window.workspaceEscape,enc=encodeURIComponent,copy=value=>structuredClone(value); const isGraph=graph=>graph?.recipe==='external_recipe'; let epoch=0,preflight=null,definitions={},reviewed=false,grant='',recipient='',grantReceipt=null; @@ -70,9 +70,27 @@ window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,isDirty, const review=root.querySelector('#external-reviewed');if(review){review.checked=false;review.disabled=true;} const button=root.querySelector('#external-grant');if(button)button.disabled=true; } + function importable(run,artifact){return isGraph(run?.graph)&&run.state==='completed'&&run.artifact_status?.status!=='unavailable'&&artifact?.role==='scientific_fields'&&typeof artifact.name==='string'&&!artifact.locked&&/^[a-f0-9]{64}$/.test(artifact.sha256||'');} + function artifactActionHTML(run,artifact){ + if(!importable(run,artifact))return ''; + return `Creates a draft. Review and freeze separately.`; + } + async function importArtifact(run,artifact){ + if(!importable(run,artifact))throw Error('Only verified scientific fields from a completed external run can be imported.'); + if(!identity()?.scopes.includes('edit'))throw Error('Editing permission is required to import a draft.'); + const origin=snapshot(),runId=run.id,name=artifact.name,expected=artifact.sha256; + let record; + try{record=await api('runs/'+enc(runId)+'/import-data/'+enc(name),{expected_sha256:expected});} + catch(error){throw Error(error.message+' If acknowledgement was lost, inspect Data for the existing content before retrying. This action never reruns the recipe.');} + if(!record||typeof record.id!=='string')throw Error('Import response unavailable. Inspect Data before retrying.'); + if(current(origin)&&getRunId()===runId){ + status('Imported draft '+record.id+'. Review and freeze remain separate.'); + try{await onImported(record.id);}catch(error){status('Draft '+record.id+' was imported. '+error.message+' Open it from Data.');} + }else status('Draft '+record.id+' was imported. Current navigation and edits are retained; open it from Data.'); + } function runHTML(run){return `

External execution custody

${esc(run.custody_phase||'No confirmed custody phase')}

Approval ${esc(run.approval_id)} · actor ${esc(run.actor_id)}

Material SHA256 ${esc(run.material_sha256)}

${run.state==='unknown'?'

Process cessation is unconfirmed. Same-boot execution may remain fenced; acknowledgement alone cannot clear custody.

':''}

${permissions} Completion is not scientific qualification.

Approved material
${esc(JSON.stringify(run.material,null,2))}
`;} window.addEventListener('tuldok-session',()=>{invalidate();if(isGraph(getGraph()))render();}); - return {isGraph,invalidate,renderPipeline,dirtyChanged,runHTML}; + return {isGraph,invalidate,renderPipeline,dirtyChanged,runHTML,artifactActionHTML,importArtifact}; }; window.WorkspaceExternalProject=function({onCreated,announce}){ diff --git a/static/workspace-real.js b/static/workspace-real.js index aaf9be9f..aa0bcd7a 100644 --- a/static/workspace-real.js +++ b/static/workspace-real.js @@ -18,7 +18,7 @@ } return value; } - window.WorkspaceReal = function ({ pipelineRoot, runsRoot, announce, openRecord, openRun }) { + window.WorkspaceReal = function ({ pipelineRoot, runsRoot, announce, openRecord, openRun, openImportedRecord }) { let definitions, releases = [], graphs = [], graph = null, saved = "", selectedNode = "train", validation = null, busy = false, loadGeneration = 0, undo = [], redo = [], view = null, timer = null, runs = [], run = null, runGeneration = 0, notice = "", noticeError = false, @@ -26,9 +26,10 @@ const memory = history.state?.realWorkspace || {}; let graphId = memory.graphId || null, runId = memory.runId || null, pendingSubmission = memory.pendingSubmission || null; - const external = WorkspaceExternal({api,action,status,getGraph:()=>graph,getIntent:()=>loadGeneration,isDirty:()=>dirty(), + const external = WorkspaceExternal({api,action,status,getGraph:()=>graph,getIntent:()=>loadGeneration,getRunId:()=>runId,isDirty:()=>dirty(), replaceGraph:document=>{if(!graphs.some(item=>item.id===document.id))graphs.push(document);adopt(document);}, changeGraph:change,render:renderPipeline,refresh:loadPipeline, + onImported:openImportedRecord, onRun:result=>{run=result;runId=result.id;predictions=null;predictionError=null;remember();status("External attempt admitted with exact material approval.");openRun();}}); function remember() { history.replaceState({ ...history.state, realWorkspace: { version: 1, graphId, runId, pendingSubmission } }, ""); @@ -312,13 +313,17 @@ ${!externalRun && !integrity && (run.current_test_exposure?.exposed || run.prior_test_exposure?.exposed) ? '

This release contains previously exposed/reused test material. Results cannot be presented as fresh holdout evidence.

' : ""} ${externalRun ? external.runHTML(run) : integrity ? integritySummaryHTML(run.graph, run.validation_summary) : ""} ${metrics ? `
${(metrics.accuracy * 100).toFixed(1)}%Validation accuracy
${metrics.examples}Validation examples
${metrics.macro_f1.toFixed(3)}Validation macro F1
` : ""} -

Stage evidence

${(run.node_attempts || []).map((stage, i) => `
${i + 1}${esc(stage.type === "train" ? trainingTitle(run.graph) : (integrity ? {release: "Frozen release", validate: `Validate ${integrity.name.toLowerCase()} integrity`, publish: "Publish integrity report"}[stage.type] : titles[stage.type]) || stage.type)}${esc(stage.state)}
`).join("")}
Bounded progress log
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}

Verified artifacts

${run.artifacts.length ? `
    ${run.artifacts.map(a => `
  • ${a.locked ? `${esc(a.name)} · test access locked` : `${esc(a.name)}`}${a.role ? esc(a.role) + " · " : ""}${a.bytes.toLocaleString()} bytes · SHA ${esc(short(a.sha256))}
  • `).join("")}
` : '

No published artifacts yet.

'}

Every artifact read verifies its frozen hash.

+

Stage evidence

${(run.node_attempts || []).map((stage, i) => `
${i + 1}${esc(stage.type === "train" ? trainingTitle(run.graph) : (integrity ? {release: "Frozen release", validate: `Validate ${integrity.name.toLowerCase()} integrity`, publish: "Publish integrity report"}[stage.type] : titles[stage.type]) || stage.type)}${esc(stage.state)}
`).join("")}
Bounded progress log
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}

Verified artifacts

${run.artifacts.length ? `
    ${run.artifacts.map(a => `
  • ${a.locked ? `${esc(a.name)} · test access locked` : `${esc(a.name)}`}${a.role ? esc(a.role) + " · " : ""}${a.bytes.toLocaleString()} bytes · SHA ${esc(short(a.sha256))}${external.artifactActionHTML(run,a)}
  • `).join("")}
` : '

No published artifacts yet.

'}

Every artifact read verifies its frozen hash.

${run.state === "completed" && !externalRun && !integrity ? `

Per-example predictions

${esc(split)}${split === "test" ? " · exposed" : ""}
${predictionsHTML()}
` : ""}
Frozen identity, source provenance and parameters
${esc(JSON.stringify({ dataset_id: run.dataset_id, run_id: run.id, release_sha256: run.release_sha256, graph_sha256: run.graph_sha256, graph: run.graph, adapter: run.adapter, runtime: run.runtime, parameters: run.parameters, prior_test_exposure: run.prior_test_exposure, current_test_exposure: run.current_test_exposure, test_exposure: run.test_exposure }, null, 2))}
`}`; }); if (notice) { const target = runsRoot.querySelector('.real-status'); target.textContent = notice; target.classList.toggle('error-panel', noticeError); } runsRoot.querySelector("#real-runs-refresh").onclick = () => loadRuns(); runsRoot.querySelectorAll("[data-real-run]").forEach(button => button.onclick = () => { split = "validation"; selectedCell = null; loadRuns(button.dataset.realRun); }); + for(const button of runsRoot.querySelectorAll('[data-import-run-artifact]'))button.onclick=()=>{ + const selected=run,artifact=selected.artifacts.find(item=>item.name===button.dataset.importRunArtifact); + action(()=>external.importArtifact(selected,artifact)); + }; runsRoot.querySelector("#real-cancel")?.addEventListener("click", () => action(async () => { run = await api("runs/" + encoded(run.id) + "/cancel", {}); status("Cancellation requested. Waiting for confirmed worker cessation."); await loadRuns(run.id, true); })); diff --git a/static/workspaces.js b/static/workspaces.js index 7baea141..2b21aeb6 100644 --- a/static/workspaces.js +++ b/static/workspaces.js @@ -36,6 +36,7 @@ window.workspaceEscape = (value) => workspaceIntent++;if(navigation)catalogImportIntent++; $("catalog-frame").contentWindow?.TuldokWorkspaceData?.cancelRecordOpen?.(); } + window.addEventListener("tuldok-session", () => invalidateCorrection()); function renderCorrection(record = catalogContext?.record) { const banner = $("real-correction-context"); banner.hidden = source !== "catalog" || !realCorrection || record?.id !== realCorrection.id; @@ -48,6 +49,7 @@ window.workspaceEscape = (value) => runsRoot: $("real-runs-root"), announce, openRun: () => { location.hash = "runs"; }, + openImportedRecord: openCatalogRecord, openRecord: async (prediction) => { const bridge = $("catalog-frame").contentWindow?.TuldokWorkspaceData; if (!bridge) { announce("Open Data once to load the current catalog editor.", true); return; } diff --git a/tests/browser_workspace_native.cjs b/tests/browser_workspace_native.cjs new file mode 100644 index 00000000..6aae30f4 --- /dev/null +++ b/tests/browser_workspace_native.cjs @@ -0,0 +1,121 @@ +'use strict'; +// Actual optional native adapter through production approval/Run/Data owners. +// Required native artifacts are explicit; there is no authored-data fallback. +const fixtureAuth=require('./auth_helpers.cjs').authenticatedFixture(),fetch=fixtureAuth.fetch; +const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); +const {spawn,execFileSync}=require('node:child_process'); +const {pageLoadTracker}=require('./browser_page_load.cjs'); +const {waitForDebugger}=require('./browser_startup.cjs'); +const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); +const root=path.resolve(__dirname,'..'),report=qaDirectory(root,'workspace-native-generation'); +function option(name,environment){const index=process.argv.indexOf(name);return index<0?process.env[environment]:process.argv[index+1];} +const binary=option('--binary','TULDOK_NATIVE_BINARY'),buildReceipt=option('--build-receipt','TULDOK_NATIVE_BUILD_RECEIPT'); +assert.ok(binary&&buildReceipt,'Actual --binary and --build-receipt are required; no fixture fallback'); +const numpy=process.env.TULDOK_TRAINING_PYTHON||process.env.NUMPY_PYTHON||'/workspace/Tuldok/.venv-training/bin/python'; +const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-native-browser-')),projectRoot=path.join(temporary,'project'); +execFileSync('python3',['adapters/rheon/project.py','--binary',binary,'--build-receipt',buildReceipt,'--destination',projectRoot],{cwd:root}); +const project=JSON.parse(fs.readFileSync(path.join(projectRoot,'project.json'))); +const sha=raw=>crypto.createHash('sha256').update(raw).digest('hex'); +const originalFiles=Object.fromEntries(fs.readdirSync(projectRoot).filter(name=>fs.statSync(path.join(projectRoot,name)).isFile()).map(name=>[name,sha(fs.readFileSync(path.join(projectRoot,name)))])); +const inspections=[],captures=[],checks=[],requests=[],errors=[],children=[],tracker=pageLoadTracker();const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms));let ws,send,evaluate,serverErrors=''; +async function until(fn,message='Timed out'){for(let i=0;i<600;i++){const value=await fn();if(value)return value;await pause(100);}throw Error(message);} +function launch(command,args,options){const child=spawn(command,args,options);children.push(child);return child;} +const serverArgs=port=>['-u','app.py','--port',String(port),'--data',path.join(temporary,'data'),'--project-root','sample='+projectRoot]; +(async()=>{ + let server=launch('python3',serverArgs(0),{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); + const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); + const base='http://127.0.0.1:'+port; + await fixtureAuth.register('http://127.0.0.1:'+port,path.join(temporary,'data'),root); + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); + const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); + ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next,timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP timeout '+method));},20000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});ws.send(JSON.stringify({id,method,params}));}); + ws.onmessage=event=>{const message=JSON.parse(event.data);tracker.observe(message);if(message.id){const task=pending.get(message.id);pending.delete(message.id);if(task)message.error?task.reject(message.error):task.resolve(message.result);}if(message.method==='Runtime.exceptionThrown')errors.push(message.params.exceptionDetails);if(message.method==='Network.requestWillBeSent')requests.push({url:message.params.request.url,method:message.params.request.method});}; + evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + const doc=frame=>frame?'document.getElementById("catalog-frame").contentDocument':'document'; + const element=(selector,frame=false)=>`${doc(frame)}.querySelector(${JSON.stringify(selector)})`,text=(selector,frame=false)=>evaluate(`${element(selector,frame)}?.textContent`); + async function click(selector,frame=false){ + const box=await until(()=>evaluate(`(async()=>{const e=${element(selector,frame)};if(!e||e.disabled||!e.getClientRects().length)return null;${frame?'document.getElementById("catalog-frame").scrollIntoView({block:"start"});':''}e.scrollIntoView({block:'center'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));const b=e.getBoundingClientRect(),x=b.x+b.width/2,y=b.y+b.height/2,hit=${doc(frame)}.elementFromPoint(x,y),f=${frame?'document.getElementById("catalog-frame").getBoundingClientRect()':'{x:0,y:0}'};return hit&&(hit===e||e.contains(hit))?{x:x+f.x,y:y+f.y}:null})()`),'Click target unavailable '+selector); + for(const type of ['mousePressed','mouseReleased'])await send('Input.dispatchMouseEvent',{type,button:'left',clickCount:1,...box}); + } + async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Escape:27,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} + async function fill(selector,value,frame=false){await click(selector,frame);await key('a','KeyA',2);await send('Input.insertText',{text:value});} + const select=(selector,value,frame=false)=>evaluate(`(()=>{const e=${element(selector,frame)};e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); + async function nav(view){await evaluate(`document.getElementById('nav-${view}').focus()`);await key('Enter');await until(()=>evaluate(`!document.getElementById('view-${view}').hidden && !document.getElementById('project-${view}-root').hidden`));await until(()=>text(`#project-${view}-root`).then(value=>value?.includes('PROJECT · INSPECTION ONLY')));} + async function capture(name,width){ + await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false}); + await evaluate(`(async()=>{const iframe=document.getElementById('catalog-frame');iframe.scrollIntoView({block:'start'});iframe.contentDocument.getElementById('scientific-inspection').scrollIntoView({block:'start'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));window.scrollBy(0,iframe.getBoundingClientRect().top);})()`); + await pause(100);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'),'Outer overflow'); + assert.ok(await evaluate(`${doc(true)}.documentElement.scrollWidth<=${doc(true)}.defaultView.innerWidth+1`),'Inspector overflow'); + const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Actual bounded native generation and custody; no model/GPU acceptance'}); + } + const check=value=>{checks.push(value);console.log('PASS',value);}; + async function reload(){const previous=(await send('Page.getFrameTree')).frameTree.frame;await send('Page.reload');await until(()=>tracker.reloaded(previous),'Main document reload not observed');} + async function openProject(manifest,name,id){await click('#open-project');await until(()=>evaluate('document.getElementById("project-open-dialog").open && !document.getElementById("project-open-submit").disabled'));await select('#project-root','sample');await fill('#project-manifest',manifest);await evaluate('document.getElementById("project-open-submit").focus()');await key('Enter');await until(()=>evaluate('document.getElementById("dataset").value==='+JSON.stringify('project:'+id)));await nav('data');assert.match(await text('#project-data-root'),new RegExp(name.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')));} + await send('Page.enable');await send('Page.setLifecycleEventsEnabled',{enabled:true});await send('Runtime.enable');await send('Network.enable'); + const offlineCanary=process.env.LD_PRELOAD?JSON.parse(execFileSync('python3',['-c',"import sys;sys.path.insert(0,'tests');from test_workspace_offline import CANARY;exec(CANARY)"],{cwd:root,encoding:'utf8'})):null; + await fixtureAuth.browser(send,'http://127.0.0.1:'+port); + await send('Page.navigate',{url:base+'/workspaces'});await until(()=>evaluate('!!document.getElementById("open-project")')); + async function captureWorkspace(name,width,selector){await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});await evaluate(`${element(selector)}.scrollIntoView({block:'start'})`);await pause(100);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'));const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Actual bounded native generation UI; no model/GPU acceptance'});} + async function api(route,body){const response=await fetch(base+route,body===undefined?{}:{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});const value=await response.json();assert.ok(response.ok,JSON.stringify({route,status:response.status,value}));return value;} + async function artifact(run,name){const item=run.artifacts.find(item=>item.name===name);assert.ok(item,name);const response=await fetch(base+item.url);assert.equal(response.status,200);const bytes=Buffer.from(await response.arrayBuffer());assert.equal(sha(bytes),item.sha256);return bytes;} + async function completed(id){return until(async()=>{const run=await api('/api/workspace/runs/'+id);assert.ok(!['failed','unknown','cancelled','interrupted'].includes(run.state),JSON.stringify({state:run.state,error:run.error,events:run.events}));return run.state==='completed'?run:null;},'Actual native generation did not complete');} + async function cold(run,label,seed,frames,observations){const directory=path.join(report,label);fs.mkdirSync(directory);for(const name of ['fields.zip','wire.bin','producer-receipt.json','native-diagnostics.txt'])fs.writeFileSync(path.join(directory,name),await artifact(run,name));const args=['tests/qualify_native_generation.py','--bundle',path.join(directory,'fields.zip'),'--wire',path.join(directory,'wire.bin'),'--seed',String(seed),'--frames',String(frames),'--report',path.join(directory,'cold.json')];if(observations){const file=path.join(directory,'inspections.json');fs.writeFileSync(file,'['+observations.join(',')+']');args.push('--inspections',file);}return JSON.parse(execFileSync(numpy,args,{cwd:root,encoding:'utf8'}));} + async function submittedGraph(graph){const preflight=await api('/api/workspace/graphs/'+graph.id+'/preflight-external',{revision:graph.revision});return api('/api/workspace/runs/submit-external',{request_key:'native-'+crypto.randomBytes(16).toString('hex'),graph_id:graph.id,graph_revision:graph.revision,expected_digest:preflight.material_sha256});} + await openProject('project.json',project.name,project.id);await nav('pipeline'); + await click('[data-project-recipe="generate"]');await until(()=>evaluate('!!document.querySelector("[data-external-input=build_receipt]")')); + await select('[data-external-input="build_receipt"]','build_receipt');await fill('#external-create-name','Actual native 32³ fields'); + await evaluate('document.getElementById("external-create-submit").focus()');await key('Enter');await until(()=>evaluate('!!document.getElementById("external-preflight")')); + await fill('#external-param-grid','33');await click('#external-save');await until(()=>text('#external-dirty').then(value=>value?.startsWith('Saved')));await click('#external-preflight'); + await until(()=>text('#real-pipeline-root .real-status').then(value=>/Invalid|bound/i.test(value||''))); + assert.deepEqual(await api('/api/workspace/runs'),[]); + await fill('#external-param-grid','32');await fill('#external-name','32³ · two frames · seed 100');await click('#external-save');await until(()=>text('#external-dirty').then(value=>value?.startsWith('Saved'))); + await click('#external-preflight');await until(()=>evaluate('!document.getElementById("external-reviewed").disabled')); + assert.match(await text('#external-material'),/Material SHA256/);await captureWorkspace('native-pipeline-1440',1440,'#external-material');await captureWorkspace('native-pipeline-390',390,'#external-material');await click('#external-reviewed');await click('#external-submit'); + await until(()=>text('#real-run-state').then(value=>value==='completed'),'Native generation UI never completed'); + const primary=(await api('/api/workspace/runs'))[0],primaryDetail=await completed(primary.id),fields=await artifact(primaryDetail,'fields.zip'); + assert.equal(primaryDetail.quality_claim,false);assert.equal(primaryDetail.material.parameters.grid,32);assert.equal(primaryDetail.material.parameters.frames,2);assert.equal(primaryDetail.material.parameters.seed,100); + await captureWorkspace('native-runs-390',390,'.external-run-custody');await captureWorkspace('native-runs-1440',1440,'.external-run-custody'); + const bundleArtifact=primaryDetail.artifacts.find(item=>item.name==='fields.zip'); + const bad=await fetch(base+'/api/workspace/runs/'+primary.id+'/import-data/fields.zip',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({expected_sha256:'0'.repeat(64)})});assert.equal(bad.status,409); + const retainedPath=path.join(temporary,'data','workspace-runs',primary.id,'fields.zip');fs.writeFileSync(retainedPath,Buffer.concat([fields,Buffer.from('tamper')])); + try{const drift=await fetch(base+'/api/workspace/runs/'+primary.id+'/import-data/fields.zip',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({expected_sha256:bundleArtifact.sha256})});assert.equal(drift.status,409);}finally{fs.writeFileSync(retainedPath,fields);} + assert.equal((await api('/api/workbench/records')).items.length,0); + check('Actual native approved generation completes; invalid grid and stale/tampered artifact imports refuse before Data writes'); + await click('[data-import-run-artifact="fields.zip"]');await until(()=>evaluate(`${doc(true)}?.getElementById('scientific-inspection')?.hidden===false`)); + await until(()=>text('#scientific-status',true).then(value=>value?.includes('Producer-declared stage:'))); + const record=(await api('/api/workbench/records')).items[0];assert.equal(record.review,'draft');assert.equal(record.content_hash,bundleArtifact.sha256);assert.equal(record.provenance.method,'run_artifact_import'); + const manifest=record.field_sequence.manifest; + assert.equal(manifest.fields.length,6);assert.equal(manifest.frames.length,2); + for(const field of manifest.fields){await select('#scientific-field',field.id,true);await select('#scientific-frame','1',true);for(const axis of ['x','y','z']){await select('#scientific-axis',axis,true);await until(()=>text('#scientific-status',true).then(value=>value?.includes(field.stage)&&value.includes(manifest.frames[1].time)));await until(()=>text('#scientific-coordinates',true).then(value=>value?.startsWith(axis.toUpperCase()+' index')));const observed=await(await fetch(base+'/api/workbench/records/'+record.id+'/scientific?'+new URLSearchParams({frame:1,field:field.id,axis,index:0}))).text();inspections.push(observed);assert.match(await text('#scientific-probe',true),/physical/);}} + await select('#scientific-field','pressure',true);await select('#scientific-frame','0',true);await select('#scientific-axis','z',true); + await capture('native-data-1440',1440);await capture('native-data-390',390); + const firstCold=await cold(primaryDetail,'seed100-two-frames',100,2,inspections);assert.equal(firstCold.arrays.length,12); + check('All six native fields and eighteen inspected planes match independent NumPy/wire bytes, MAC placement and diagnostics'); + await send('Emulation.setDeviceMetricsOverride',{width:1440,height:900,deviceScaleFactor:1,mobile:false}); + await fill('#scientific-note','Reviewed actual bounded native generation and exact field custody; no model, GPU or general physics acceptance.',true);await select('#record-review','human_reviewed',true); + await evaluate(`${element('#editor button:not([type="button"])',true)}.focus()`);await key('Enter');await until(async()=>(await api('/api/workbench/records/'+record.id)).review==='human_reviewed'); + await until(()=>text('#records',true).then(value=>value?.includes('revision 2')));await click('#select-page',true);await click('.embed-toolbar button',true);await evaluate(`${element('#release-form',true)}.closest('details').open=true`); + await fill('#train','100',true);await fill('#validation','0',true);await fill('#test','0',true);await click('#preview-release',true);await until(()=>evaluate(`${element('#freeze-release',true)}?.disabled===false`));await click('#freeze-release',true);await until(()=>evaluate(`!!${doc(true)}.querySelector('#release-result a')`)); + const download=await evaluate(`${doc(true)}.querySelector('#release-result a').href`),frozen=Buffer.from(await(await fetch(download)).arrayBuffer()),releasePath=path.join(report,'native-release.zip'),recovered=path.join(report,'recovered.zip');fs.writeFileSync(releasePath,frozen); + const released=JSON.parse(execFileSync('python3',['-c',"import json,sys,zipfile;from pathlib import Path;from scientific_release import validate_release;r=validate_release(Path(sys.argv[1]));z=zipfile.ZipFile(sys.argv[1]);m=json.loads(z.read('manifest.json'));assert len(m['records'])==1;row=m['records'][0];assert row['review']=='human_reviewed';Path(sys.argv[2]).write_bytes(z.read(row['asset']));print(json.dumps(r))",releasePath,recovered],{cwd:root,encoding:'utf8'}));assert.deepEqual(fs.readFileSync(recovered),fields); + const frozenCold=JSON.parse(execFileSync(numpy,['tests/qualify_native_generation.py','--bundle',recovered,'--wire',path.join(report,'seed100-two-frames','wire.bin')],{cwd:root,encoding:'utf8'}));assert.equal(frozenCold.bundle_sha256,firstCold.bundle_sha256); + check('Separate explicit human review and canonical freeze preserve the exact generated bundle and verifiable run provenance'); + const variants=[]; + for(const [seed,frames] of [[101,2],[100,1]]){let graph=await api('/api/workspace/graphs/'+primaryDetail.graph.id+'/clone',{name:`Native seed ${seed}, ${frames} frames`});graph.nodes.find(node=>node.type==='execute').parameters={grid:32,seed,frames};graph=await api('/api/workspace/graphs/'+graph.id,{revision:graph.revision,document:graph});const result=await completed((await submittedGraph(graph)).id);const output=result.artifacts.find(item=>item.name==='fields.zip');const imported=await api('/api/workspace/runs/'+result.id+'/import-data/fields.zip',{expected_sha256:output.sha256,name:`Native seed ${seed}, ${frames} frames`});assert.equal(imported.review,'draft');variants.push({run:result.id,record_id:imported.id,groups:imported.groups,cold:await cold(result,`seed${seed}-${frames}-frames`,seed,frames)});} + assert.notDeepEqual(variants[0].cold.family,firstCold.family);assert.deepEqual(variants[1].cold.family,firstCold.family);const protectedFamily=record.groups.filter(value=>value.startsWith('field-sequence-family:'));assert.equal(protectedFamily.length,1);assert.ok(variants[1].groups.includes(protectedFamily[0]));assert.ok(!variants[0].groups.includes(protectedFamily[0])); + for(const original of firstCold.arrays.filter(item=>item.frame===0)){assert.equal(variants[1].cold.arrays.find(item=>item.field===original.field).payload_sha256,original.payload_sha256);} + assert.ok(firstCold.arrays.some((item,index)=>item.payload_sha256!==variants[0].cold.arrays[index].payload_sha256)); + check('A second native seed changes actual arrays and family; shorter frame count preserves original initial-condition family and frame bytes'); + let cancelGraph=await api('/api/workspace/graphs/'+primaryDetail.graph.id+'/clone',{name:'Cancelled native request'});cancelGraph.nodes.find(node=>node.type==='execute').parameters.seed=102;cancelGraph=await api('/api/workspace/graphs/'+cancelGraph.id,{revision:cancelGraph.revision,document:cancelGraph});const cancelRun=await submittedGraph(cancelGraph);await api('/api/workspace/runs/'+cancelRun.id+'/cancel',{}); + const cancelled=await until(async()=>{const value=await api('/api/workspace/runs/'+cancelRun.id);return value.state==='cancelled'?value:null;});const refused=await fetch(base+'/api/workspace/runs/'+cancelRun.id+'/import-data/fields.zip',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({expected_sha256:bundleArtifact.sha256})});assert.equal(refused.status,409);assert.equal((await api('/api/workbench/records')).items.length,3); + check('Cancellation publishes no importable partial scientific artifact'); + server.kill('SIGINT');await until(()=>server.exitCode!==null);server=launch('python3',serverArgs(port),{cwd:root,stdio:['ignore','pipe','pipe']});let reopened='';server.stdout.on('data',chunk=>reopened+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk);await until(()=>reopened.includes('Tuldok:'));await reload();await until(()=>text('#scientific-status',true).then(value=>value?.includes('Producer-declared stage:'))); + assert.deepEqual(Buffer.from(await(await fetch(download)).arrayBuffer()),frozen);assert.deepEqual(await artifact(await api('/api/workspace/runs/'+primary.id),'fields.zip'),fields);assert.equal((await api('/api/workbench/records/'+record.id)).review,'human_reviewed'); + for(const [name,hash] of Object.entries(originalFiles))assert.equal(sha(fs.readFileSync(path.join(projectRoot,name))),hash); + assert.deepEqual(errors,[]);assert.ok(!serverErrors.includes('Traceback'),serverErrors);check('Server restart retains approved Run, reviewed data, exact frozen release and unchanged project sources'); + fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({source_commit:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),dirty:!!execFileSync('git',['status','--porcelain'],{cwd:root,encoding:'utf8'}).trim(),qualification:'Actual optional native generator through approved production Runs; exact field custody and bounded numerical observations',model_or_training_acceptance:false,general_physics_acceptance:false,gpu_or_performance_acceptance:false,checks,captures,cancelled_run:{id:cancelled.id,state:cancelled.state,custody_phase:cancelled.custody_phase,events:cancelled.events},run_id:primary.id,record_id:record.id,build_receipt_sha256:sha(fs.readFileSync(buildReceipt)),binary_sha256:sha(fs.readFileSync(binary)),cold:firstCold,variants,released,offline_canary:offlineCanary,browser:(await send('Browser.getVersion')).product},null,2));console.log('PASS native generation browser:',report); +})().catch(async error=>{console.error(error);console.error('Application stderr:',serverErrors);if(evaluate)try{console.error(await evaluate('JSON.stringify({url:location.href,body:document.body.innerText.slice(-6000),frame:document.getElementById("catalog-frame")?.contentDocument?.body.innerText.slice(-4000)})'));}catch{}process.exitCode=1;}).finally(()=>{ws?.close();for(const child of children.reverse())child.kill('SIGTERM');}); diff --git a/tests/qualify_native_generation.py b/tests/qualify_native_generation.py new file mode 100644 index 00000000..d72560d0 --- /dev/null +++ b/tests/qualify_native_generation.py @@ -0,0 +1,201 @@ +"""Independent NumPy/wire oracle for actual bounded native generation. + +This module never imports the optional producer adapter or launches a producer. +It verifies cold artifacts from approved Runs; no model/GPU/general physics claim. +""" +import argparse +import hashlib +import io +import json +import math +from pathlib import Path +import struct +import zipfile + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def decode_wire(raw, grid, frames, seed): + offset = 0 + def take(length): + nonlocal offset + assert 0 <= length <= len(raw) - offset, 'Truncated native wire' + value = raw[offset:offset + length] + offset += length + return value + def message(): + length, = struct.unpack('= 0 and np.max(tracer) <= 1 + integral = float(np.sum(tracer) / grid ** 3) + assert math.isclose(integral, report['tracer_integral'], rel_tol=1e-12, abs_tol=1e-15) + diagnostics.append({'frame': frame_index, 'actual_divergence_max': maximum, + 'wall_normal_velocity_zero': True, 'tracer_integral': integral, + 'tracer_integral_tolerance': {'relative': 1e-12, 'absolute': 1e-15}}) + source_wire = [] + sources = {} + for item in manifest['sources']: + raw = archive.read(item['path']) + assert len(raw) == item['bytes'] and sha(raw) == item['sha256'] + sources[Path(item['path']).name] = raw + if Path(item['path']).name == 'wire.bin': + source_wire.append(raw) + assert source_wire == [wire], 'Original native wire absent or changed' + receipt_raw = Path(wire_path).with_name('producer-receipt.json').read_bytes() + assert sources['producer-receipt.json'] == receipt_raw + receipt = json.loads(receipt_raw) + assert receipt['controls'] == {'grid': grid, 'frames': frames, 'seed': seed, 'method': 'jacobi'} + assert receipt['ready'] == ready and receipt['frames'] == [report for report, _ in native] + assert receipt['wire_sha256'] == sha(wire) and receipt['wire_bytes'] == len(wire) + assert receipt['stdin_hex'] == bytes([2] * frames + [0]).hex() + assert sha(sources['build-receipt.json']) == receipt['build_receipt_sha256'] + assert json.loads(sources['build-receipt.json'])['binary_sha256'] == receipt['binary_sha256'] + assert manifest['provenance']['source_id'] == sha(receipt_raw) + mappings = receipt['loaded_mappings'] + assert 'sealed-native' in mappings and 'sealed-loader' in mappings + policy_raw = sources['native-policy.json'] + assert sha(policy_raw) == receipt['native_policy_sha256'] + policy = json.loads(policy_raw) + assert policy['source']['sha256'] == receipt['binary_sha256'] + declared_mappings = {item['path'] for item in policy['runtime_files']} - {policy['loader']['path']} + assert set(mappings) == declared_mappings | {'sealed-native', 'sealed-loader'} + assert all(value['inode'] > 0 and ':' in value['device'] for value in mappings.values()) + assert all(name in ('sealed-native', 'sealed-loader') or name.startswith('/usr/lib/x86_64-linux-gnu/') for name in mappings) + diagnostics_raw = Path(wire_path).with_name('native-diagnostics.txt').read_bytes() + assert sha(diagnostics_raw) == receipt['diagnostics_sha256'] + if inspections is not None: + for observed in json.loads(Path(inspections).read_bytes()): + array = arrays[observed['frame'], observed['field']] + axis, index = observed['axis'], observed['index'] + expected = {'x': lambda: array[:, :, index], 'y': lambda: array[:, index, :], + 'z': lambda: array[index, :, :]}[axis]() + assert np.array(observed['values'], dtype=array.dtype).tobytes() == expected.tobytes() + field = next(field for field in manifest['fields'] if field['id'] == observed['field']) + assert observed['stage'] == field['stage'] + assert observed['time'] == manifest['frames'][observed['frame']]['time'] + assert observed['dtype'] == field['dtype'] and observed['units'] == field['units'] + row_axis, column_axis = {'x': ('z', 'y'), 'y': ('z', 'x'), 'z': ('y', 'x')}[axis] + assert (observed['row_axis'], observed['column_axis']) == (row_axis, column_axis) + offset = np.full(3, .5) + if field['centering'] != 'cell': + offset['xyz'.index(field['centering'][-1])] = 0 + offset['xyz'.index(axis)] += index + assert np.array_equal(np.array(observed['physical_origin']), offset / grid) + for dimension, name in ((row_axis, 'physical_row_step'), (column_axis, 'physical_column_step')): + step = np.zeros(3); step['xyz'.index(dimension)] = 1 / grid + assert np.array_equal(np.array(observed[name]), step) + + return {'scope': 'Actual native generation custody and bounded full-grid numerical observations', + 'numpy': np.__version__, 'bundle_sha256': sha(bundle_raw), 'wire_sha256': sha(wire), + 'grid': grid, 'frames': frames, 'seed': seed, 'family': manifest['family'], + 'arrays': checks, 'diagnostics': diagnostics, 'original_wire_exact': True, + 'model_or_training_acceptance': False, 'general_physics_acceptance': False, + 'gpu_or_performance_acceptance': False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--bundle', required=True) + parser.add_argument('--wire', required=True) + parser.add_argument('--grid', type=int, default=32) + parser.add_argument('--frames', type=int, default=2) + parser.add_argument('--seed', type=int, default=100) + parser.add_argument('--inspections') + parser.add_argument('--report') + args = parser.parse_args() + result = qualify(args.bundle, args.wire, args.grid, args.frames, args.seed, args.inspections) + encoded = json.dumps(result, indent=2) + '\n' + if args.report: + Path(args.report).write_text(encoded) + print(encoded, end='') + + +if __name__ == '__main__': + main() diff --git a/tests/test_recipe_contract.py b/tests/test_recipe_contract.py index ef7d9255..5b3b4c98 100644 --- a/tests/test_recipe_contract.py +++ b/tests/test_recipe_contract.py @@ -4,6 +4,7 @@ import json from pathlib import Path import tempfile +import struct import unittest from unittest.mock import patch import uuid @@ -37,6 +38,36 @@ def test_parameters_and_closed_tokens(self): changed=copy.deepcopy(value); changed['outputs'][0]['path']=name with self.assertRaises(WorkbenchError): contract.decode_definition(json.dumps(changed).encode()) + def test_strict_native_elf_rejects_headers_paths_and_loader_extensions(self): + raw=Path('/usr/bin/true').read_bytes() + needed=contract.strict_native_elf(raw,executable=True) + self.assertIn('libc.so.6',needed) + offset=struct.unpack_from('{let resolve,reject;const promise=new Promise((a,b)=>{resolve=a;reject=b});return {promise,resolve,reject};}; function fixture(){ - let actor={actor_id:'owner',credential_id:'session1',kind:'owner',scopes:['approve-execution']},intent=0,handler; - const history={state:{},replaceState(value){this.state=value;}},listeners={},nodes={},calls=[],adopted=[],opened=[],notices=[]; + let actor={actor_id:'owner',credential_id:'session1',kind:'owner',scopes:['approve-execution','edit']},intent=0,runId='run1',handler; + const history={state:{},replaceState(value){this.state=value;}},listeners={},nodes={},calls=[],adopted=[],opened=[],imported=[],notices=[]; const graph={id:'g',revision:1,name:'neutral',recipe:'external_recipe',nodes:[{type:'input',parameters:{}},{type:'execute',parameters:{mode:'complete'}},{type:'output',parameters:{}}]}; const node=id=>nodes[id]||(nodes[id]={value:'owner',addEventListener(type,fn){this[type]=fn;}}); const root={innerHTML:'',querySelector:node,querySelectorAll:()=>[]}; @@ -13,11 +13,11 @@ function fixture(){ const context={window,history,crypto,structuredClone,confirm:()=>true};vm.runInNewContext(source,context); const api=async(route,body)=>{calls.push({route,body:structuredClone(body)});if(route.endsWith('preflight-external'))return {valid:true,graph_revision:1,material_sha256:'a'.repeat(64),parameter_definitions:{},material:{},outputs:[],runtime:{}};return handler(route,body);}; let latest; - const ui=window.WorkspaceExternal({api,action:fn=>latest=fn(),status:value=>notices.push(value),getGraph:()=>graph,getIntent:()=>intent,isDirty:()=>false,replaceGraph:value=>adopted.push(value),changeGraph:()=>{},render:()=>{},refresh:()=>{},onRun:value=>opened.push(value)}); + const ui=window.WorkspaceExternal({api,action:fn=>latest=fn(),status:value=>notices.push(value),getGraph:()=>graph,getIntent:()=>intent,getRunId:()=>runId,isDirty:()=>false,replaceGraph:value=>adopted.push(value),changeGraph:()=>{},render:()=>{},refresh:()=>{},onRun:value=>opened.push(value),onImported:async id=>imported.push(id)}); const render=()=>ui.renderPipeline(root,{graph,graphs:[graph],busy:false,choose:()=>{}}); render(); - return {history,nodes,calls,adopted,opened,notices,root,render,ui,setHandler:value=>handler=value, - navigate(){intent++;},session(){actor={...actor,credential_id:'session2'};listeners['tuldok-session']();},other(){actor={...actor,actor_id:'other'};listeners['tuldok-session']();}, + return {history,nodes,calls,adopted,opened,imported,notices,root,render,ui,setHandler:value=>handler=value, + navigate(){intent++;},selectRun(){runId='run2';},revokeEdit(){actor={...actor,scopes:[]};},session(){actor={...actor,credential_id:'session2'};listeners['tuldok-session']();},other(){actor={...actor,actor_id:'other'};listeners['tuldok-session']();}, async review(){await node('#external-preflight').onclick();node('#external-reviewed').change({target:{checked:true}});}, click(id){node(id).onclick?.();return latest;},event(id){node(id).click();return latest;}}; } @@ -40,5 +40,17 @@ function fixture(){ } } const h=fixture();await h.review();h.setHandler(async()=>{throw Object.assign(Error('input changed before admission'),{status:409,code:'stale'});});await assert.rejects(h.click('#external-submit'));assert.equal(h.history.state.externalSubmission,null); + const artifact={name:'fields.zip',role:'scientific_fields',sha256:'b'.repeat(64)},run={id:'run1',state:'completed',graph:{recipe:'external_recipe'}}; + const imported=fixture();imported.setHandler(async()=>({id:'draft1'}));assert.match(imported.ui.artifactActionHTML(run,artifact),/Import into Data/); + await imported.ui.importArtifact(run,artifact);assert.deepEqual(imported.imported,['draft1']);assert.deepEqual(imported.calls[0],{route:'runs/run1/import-data/fields.zip',body:{expected_sha256:artifact.sha256}}); + for(const change of ['navigate','session','selectRun']){ + const f=fixture(),held=deferred();f.setHandler(()=>held.promise);const result=f.ui.importArtifact(run,artifact);f[change]();held.resolve({id:'saved-draft'});await result;assert.deepEqual(f.imported,[]);assert.ok(f.notices.some(value=>value.includes('saved-draft'))); + } + const blocked=fixture();blocked.setHandler(()=>{throw Error('Must not call');}); + for(const state of ['running','cancelled','unknown']){assert.equal(blocked.ui.artifactActionHTML({...run,state},artifact),'');await assert.rejects(blocked.ui.importArtifact({...run,state},artifact));} + assert.equal(blocked.ui.artifactActionHTML({...run,artifact_status:{status:'unavailable'}},artifact),'');assert.equal(blocked.ui.artifactActionHTML(run,{...artifact,role:'result'}),''); + blocked.revokeEdit();assert.match(blocked.ui.artifactActionHTML(run,artifact),/disabled/);await assert.rejects(blocked.ui.importArtifact(run,artifact),/permission/);assert.equal(blocked.calls.length,0); + const uncertain=fixture();uncertain.setHandler(async()=>{throw Error('acknowledgement lost');});await assert.rejects(uncertain.ui.importArtifact(run,artifact),/inspect Data/);assert.equal(uncertain.calls.length,1);assert.deepEqual(uncertain.imported,[]); + console.log('PASS scientific artifact import controller: exact hash/route, draft navigation, session/view/run-selection fences, non-completed/unavailable role refusal, edit permission and no replay after uncertain acknowledgement. Mock transport only.'); console.log('PASS external controller: actor-bound unresolved key survives ambiguous→403; no cross-actor request; late save/reload/clone/grant/submit preserve navigation/session; initial authoritative rejection clears key. Mock transport only.'); })().catch(error=>{console.error(error);process.exitCode=1;});