From d191e681011756d7d3204604a410fe5619b5c559 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 13 Oct 2026 20:23:40 +0000 Subject: [PATCH 1/3] Prepare autonomous evaluation with retained fields and bounded artifact streaming --- .github/workflows/tests.yml | 1 + adapters/rheon/evaluate.py | 391 +++ adapters/rheon/evaluation_project.py | 74 + adapters/rheon/native_transport.py | 12 +- app.py | 24 +- docs/contracts/captured-cpu-runtime.md | 12 + docs/contracts/external-recipes.md | 47 + docs/contracts/rheon-evaluation.md | 129 + .../plans/generic-scientific-projects/plan.md | 4 +- .../reports/p5c-admission.md | 485 ++++ .../reports/p5c/assembly.json | 14 + .../p5c/duplicate-cleanup-inventory.json | 2282 +++++++++++++++++ .../p5c/duplicate-cleanup-receipt.json | 768 ++++++ .../reports/p5c/installer-cleanup.json | 31 + pipeline.py | 6 +- recipe_contract.py | 136 +- recipe_process.py | 82 +- runs.py | 257 +- scientific_fields.py | 43 +- static/workspace-external.js | 57 +- static/workspace-real.js | 4 +- tests/browser_workspace_rheon_evaluation.cjs | 93 + tests/qualify_rheon_evaluation.py | 245 ++ tests/test_external_artifacts.py | 165 ++ tests/test_external_progress.py | 25 +- tests/test_large_recipe_contract.py | 83 + tests/test_recipe_process.py | 77 + tests/test_rheon_evaluation.py | 185 ++ tests/test_run_recipe_inputs.py | 92 + tests/test_workspace_external_controller.cjs | 36 +- 30 files changed, 5705 insertions(+), 155 deletions(-) create mode 100644 adapters/rheon/evaluate.py create mode 100644 adapters/rheon/evaluation_project.py create mode 100644 docs/contracts/rheon-evaluation.md create mode 100644 docs/plans/generic-scientific-projects/reports/p5c-admission.md create mode 100644 docs/plans/generic-scientific-projects/reports/p5c/assembly.json create mode 100644 docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-inventory.json create mode 100644 docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-receipt.json create mode 100644 docs/plans/generic-scientific-projects/reports/p5c/installer-cleanup.json create mode 100644 tests/browser_workspace_rheon_evaluation.cjs create mode 100644 tests/qualify_rheon_evaluation.py create mode 100644 tests/test_external_artifacts.py create mode 100644 tests/test_large_recipe_contract.py create mode 100644 tests/test_rheon_evaluation.py create mode 100644 tests/test_run_recipe_inputs.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 799639cd..2f8d1100 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,6 +50,7 @@ jobs: - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_pointclouds.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_cpu_training.py + - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_evaluation.py - run: python -m unittest discover -s tests - run: node --check static/auth.js - run: node tests/test_auth_helpers.cjs diff --git a/adapters/rheon/evaluate.py b/adapters/rheon/evaluate.py new file mode 100644 index 00000000..4059cd74 --- /dev/null +++ b/adapters/rheon/evaluate.py @@ -0,0 +1,391 @@ +"""Original evaluation numerics with explicit custody and full-field retention hooks.""" +import argparse +from contextlib import contextmanager +import hashlib +import io +import math +import os +from pathlib import Path +import shutil +import statistics +import struct +import sys +import tempfile +import zipfile + +if __package__: + from . import cpu_compare, generate, native_transport +else: + import cpu_compare + import generate + import native_transport + +SEEDS = (900,901,902,903) +METHODS = ('jacobi','sgs','learned') +CAPTURE_METHODS = ('jacobi','learned') +WIRE_LIMIT = 170*1024*1024 +CHUNK_LIMIT = 40*1024*1024 +TOTAL_LIMIT = 3500000000 +FIELDS = [('rhs','=0)&(array<=1)).all().item()),'Invalid native field values.') + if name=='rhs' and inference_rhs is not None:require(raw==inference_rhs,'Learned RHS request differs from retained frame RHS.') + arrays.append({'id':name,'dtype':dtype,'centering':center,'stage':stage,'shape_zyx':dimensions,'raw':raw}) + return report,arrays + + +def write_chunk(path,method,seed,ready,frames,wire,start,end,identity): + members={};manifest={'schema_version':1,'format':'scientific_field_sequence_v1', + 'grid':{'shape_xyz':[32]*3,'origin_xyz':[0,0,0],'spacing_xyz':ready['spacing'], + 'orientation':[[1,0,0],[0,1,0],[0,0,1]],'units':'native length units (unspecified)'}, + 'fields':[{'id':name,'label':name.replace('_',' '),'dtype':dtype,'centering':center,'stage':stage, + 'units':'native units (unspecified)'} for name,dtype,center,stage in FIELDS], + 'frames':[],'time_units':'native time units (unspecified)', + 'family':{'namespace':'rheon-trajectory-probe-scene-v1','id':generate.sha(encode({'scene':'rheon-trajectory-probe-source-v1','grid':32,'seed':seed}))}, + 'sources':[],'provenance':{'producer':'Optional Rheon autonomous evaluation; '+method,'source_id':''}} + for report,arrays in frames: + frame={'time':str(report['time']),'arrays':[]} + for array in arrays: + name=f"arrays/{report['frame']:03d}-{array['id']}.npy" + data=generate.npy(array['raw'],array['dtype'],array['shape_zyx']);members[name]=data + frame['arrays'].append({'field':array['id'],'path':name,'bytes':len(data),'sha256':generate.sha(data)}) + manifest['frames'].append(frame) + trace={'schema_version':1,'method':method,'seed':seed,'ready':ready,'frame_start':frames[0][0]['frame'], + 'frame_end':frames[-1][0]['frame'],'stdout_start':start,'stdout_end':end, + 'stdout_sha256':generate.sha(wire),'includes_ready':start==0,'reports':[f[0] for f in frames], + 'learned_rhs_request_before_report':method=='learned','identity':identity} + trace_raw=encode(trace);manifest['provenance']['source_id']=generate.sha(trace_raw) + for name,raw in [('sources/wire.bin',wire),('sources/evaluation-trace.json',trace_raw)]: + members[name]=raw;manifest['sources'].append({'path':name,'bytes':len(raw),'sha256':generate.sha(raw), + 'description':'Exact native stdout segment and evaluation identity; retained source bytes'}) + members['manifest.json']=encode(manifest) + with zipfile.ZipFile(path,'x',compression=zipfile.ZIP_STORED) as archive: + for name,raw in members.items(): + info=zipfile.ZipInfo(name,(1980,1,1,0,0,0));info.compress_type=zipfile.ZIP_STORED + archive.writestr(info,raw) + with path.open('rb') as handle:os.fsync(handle.fileno()) + require(path.stat().st_size<=CHUNK_LIMIT,'Scientific evaluation chunk exceeds40MiB.') + # Verify every retained member before the corresponding wire can be removed. + with zipfile.ZipFile(path) as archive: + require(set(archive.namelist())==set(members),'Retained chunk inventory changed.') + for name,raw in members.items():require(archive.read(name)==raw,'Retained chunk bytes differ.') + return {'path':path.name,'sha256':file_hash(path),'bytes':path.stat().st_size,'frame_start':trace['frame_start'], + 'frame_end':trace['frame_end'],'stdout_start':start,'stdout_end':end,'stdout_sha256':trace['stdout_sha256']} + + +def retain_trajectory(tee,method,seed,states,reports,output,identity): + require(tee.closed and len(states)==len(reports)==120,'Incomplete closed accuracy trajectory.') + require(tee.path.stat().st_size==tee.count and file_hash(tee.path)==tee.digest.hexdigest(),'Captured wire identity differs.') + chunks=[] + with tee.path.open('rb') as handle: + ready=message(handle) + require(ready.get('schema')=='rheon-trajectory-v1' and ready.get('n')==32 and ready.get('method')==method + and ready.get('seed')==seed and ready.get('spacing')==[1/32]*3 and ready.get('dt')==1/60 + and ready.get('density')==1.25 and ready.get('face_lengths')==[33792]*3,'Accuracy READY differs.') + start=0 + for number,first in enumerate(range(0,120,16)): + frames=[checked_frame(handle,method,i+1,states[i],reports[i]) for i in range(first,min(first+16,120))] + end=handle.tell();handle.seek(start);raw=native_transport.exact(handle,end-start) + path=output/f'fields-{method}-{seed}-{number:03d}.zip' + chunks.append(write_chunk(path,method,seed,ready,frames,raw,start,end,identity));start=end + require(not handle.read(1) and start==tee.count,'Trailing or unretained native wire.') + index={'schema_version':1,'method':method,'seed':seed,'stdout_bytes':tee.count,'stdout_sha256':tee.digest.hexdigest(), + 'frames':120,'chunks':chunks,'identity':identity} + target=output/f'wire-{method}-{seed}.json';target.write_bytes(encode(index)) + with target.open('rb') as handle:os.fsync(handle.fileno()) + require(generate.decode(target.read_bytes())==index,'Retained wire index differs.') + directory_fd=os.open(output,os.O_RDONLY|os.O_DIRECTORY) + try:os.fsync(directory_fd) + finally:os.close(directory_fd) + tee.path.unlink() # Only this verified task-created redundant tee; all ranges remain in chunks. + return index + + +def validate_inputs(raws,evidence,snapshot): + require(type(evidence) is dict and set(evidence)==set(INPUTS)|{'build_receipt'},'Input evidence slots differ.') + shared=None + for name,(role,limit) in INPUTS.items(): + entry=evidence[name] + require(type(entry) is dict and set(entry)=={'kind','source'} and entry['kind']=='run_artifact','Completed Run artifact required.') + source=entry['source'] + require(type(source) is dict and type(source.get('schema_version')) is int and source['schema_version']==1 and source.get('artifact_role')==role + and source.get('artifact_bytes')==len(raws[name])<=limit + and source.get('measured_sha256')==generate.sha(raws[name]),'Artifact input custody/role differs.') + origin={key:source.get(key) for key in ('catalog_id','run_id','attempt_id','request_sha256','material_sha256', + 'custody_receipt_sha256','custody_identity','custody_receipt_base64')} + require(all(value is not None for value in origin.values()),'Incomplete originating Run identity.') + if shared is None:shared=origin + require(shared==origin,'Training inputs do not belong to one completed Run.') + freeze=generate.decode(raws['freeze']);result=generate.decode(raws['training_result']);inputs=generate.decode(raws['training_inputs']) + require(result.get('status')=='verified' and result.get('freeze_sha256')==generate.sha(raws['freeze']) + and result.get('checkpoint_sha256')==freeze.get('checkpoint_sha256')==generate.sha(raws['checkpoint']) + and result.get('input_receipt_sha256')==generate.sha(raws['training_inputs']) + and result.get('training_data_sha256')==freeze.get('training_data_sha256') + and result.get('releases')==freeze.get('training_releases')==inputs.get('releases'), 'Training artifact payload hashes disagree.') + require(freeze.get('schema')=='rheon-trajectory-freeze-v1' and freeze.get('n')==32 and freeze.get('device')=='cpu' + and freeze.get('test_frames')==120 and freeze.get('test_seeds')==list(SEEDS) + and freeze.get('benchmark_repetitions')==5 and freeze.get('methods')==list(METHODS) + and freeze.get('training_frames')==8 and freeze.get('validation_frames')==60, + 'Fresh immutable evaluation configuration differs.') + require(result.get('source')==freeze.get('source') and result.get('training')==freeze.get('training') + and result.get('validation')==freeze.get('validation') and result.get('ablation')==freeze.get('ablation') + == 'untrained_unit_gains_rejected_by_physics_gate','Training result/freeze evidence differs.') + source=freeze['source'] + require(source.get('source_sha256')==cpu_compare.SOURCES and source.get('binary_sha256')==cpu_compare.BINARY_SHA256 + and source.get('runtime_snapshot_sha256')==snapshot and source.get('device')=='cpu' + and type(source.get('threads')) is int and 1<=source['threads']<=8 + and source.get('build_receipt_sha256')==generate.sha(raws['build_receipt'])==cpu_compare.BUILD_RECEIPT_SHA256, + 'Training source/runtime/build identity differs.') + require(inputs.get('shape')==[96,32768] and inputs.get('dtype')=='0,'Invalid timing.') + for i,report in enumerate(record['reports'],1): + require(report.get('frame')==i and math.isclose(finite(report.get('time')),i/60,rel_tol=1e-12,abs_tol=1e-12),'Benchmark frame/time differs.') + finite(report.get('relative_residual'),1e-8);finite(report.get('actual_divergence_max'),1e-5);finite(report.get('courant'),1) + require(record['method']!='learned' or report.get('iterations')==0,'Learned benchmark used correction.') + medians={method:statistics.median(r['wall_ms_including_native_startup_teardown'] for r in records if r['method']==method) for method in METHODS} + require(result.get('median_120_frame_wall_ms')==medians,'Benchmark medians differ.') + for method in ('jacobi','sgs'):require(result.get('speedup_vs_'+method)==medians[method]/medians['learned'],'Benchmark ratio differs.') + accuracy=result.get('accuracy',[]) + require([v.get('seed') for v in accuracy]==list(SEEDS),'Accuracy trajectory coverage differs.') + for case in accuracy: + require(case.get('frames')==120 and len(case.get('errors',[]))==120,'Accuracy frames missing.') + for field in ('relative_velocity_l2','relative_tracer_l2'): + maximum=max(finite(frame.get(field),1e-5) for frame in case['errors']) + require(maximum<1e-5 and case.get('max_'+field)==maximum,'Accuracy maximum invalid.') + for frame in case['errors']: + finite(frame.get('centered_relative_pressure_l2'));finite(frame.get('tracer_rmse')) + finite(case.get('max_ai_divergence'),1e-5) + require(finite(result.get('python_peak_rss_bytes'))<8000000000 + and result.get('gpu_peak_allocated_bytes')==result.get('gpu_peak_reserved_bytes')==0,'Inference memory/device gate failed.') + finite(result.get('model_warmup_s')) + + +def run(args): + root=Path(__file__).resolve().parent;output=Path(args.output) + require(output.is_dir() and not output.is_symlink() and not any(output.iterdir()),'Empty output directory required.') + require(generate.HASH.fullmatch(args.runtime_snapshot_sha256),'Runtime snapshot digest required.') + raws={name:generate.capture(getattr(args,name),limit)[0] for name,(_role,limit) in INPUTS.items()} + raws['build_receipt']=generate.capture(args.build_receipt,1024*1024)[0] + evidence=generate.decode(generate.capture(args.input_evidence,1024*1024)[0]) + freeze,origin=validate_inputs(raws,evidence,args.runtime_snapshot_sha256) + for name,digest in cpu_compare.SOURCES.items():require(generate.sha(generate.capture(root/'lab/scripts'/name,65536)[0])==digest,'Original script changed.') + binary=root/'lab/target/release/rheon-trajectory-parallel' + require(generate.sha(generate.capture(binary,8*1024*1024)[0])==cpu_compare.BINARY_SHA256,'Evaluation native changed.') + os.environ['CUDA_VISIBLE_DEVICES']='';os.environ['RAYON_NUM_THREADS']=str(freeze['source']['threads']) + sys.path.insert(0,str(root/'lab/scripts')) + import numpy as np + import torch + import trajectory_experiment as exp + require(torch.__version__==freeze['source']['torch'] and np.__version__==freeze['source']['numpy'],'Frozen framework versions differ.') + torch.set_num_threads(freeze['source']['threads']);torch.set_num_interop_threads(1) + checkpoint=torch.load(io.BytesIO(raws['checkpoint']),weights_only=True,map_location='cpu') + require(checkpoint.get('schema')=='rheon-modal-trajectory-v1' and checkpoint.get('n')==32 + and tuple(checkpoint.get('training_seeds',()))==tuple(range(100,112)),'Fresh checkpoint contract differs.') + state=checkpoint['state_dict'] + require(set(state)=={'gains','basis'} and tuple(state['gains'].shape)==(32767,) and tuple(state['basis'].shape)==(32,32) + and all(v.dtype==torch.float64 and torch.isfinite(v).all().item() for v in state.values()) + and (state['gains']>0).all().item(),'Checkpoint state invalid.') + del checkpoint,state + identity={'checkpoint_sha256':generate.sha(raws['checkpoint']),'freeze_sha256':generate.sha(raws['freeze']), + 'runtime_snapshot_sha256':args.runtime_snapshot_sha256,'binary_sha256':cpu_compare.BINARY_SHA256, + 'source_sha256':cpu_compare.SOURCES,'training_origin':origin} + policy=generate.capture(args.native_policy,1024*1024)[0] + transport=native_transport.Transport(binary,policy,evaluation_mode=True) + exp.BINARY=binary;exp.subprocess=transport + exp.provenance=lambda:dict(freeze['source']) # All returned identity components checked above against captured bytes/runtime. + original_compare=exp.compare;original_rollout=exp.rollout;indices=[];timed=0 + with private_workspace(output.parent) as directory: + private=Path(directory);run_dir=private/'run';run_dir.mkdir();wire_dir=private/'wire';wire_dir.mkdir() + (run_dir/'model.pt').write_bytes(raws['checkpoint']);(run_dir/'freeze.json').write_bytes(raws['freeze']) + def compare(n,seed,frames,model): + require(n==32 and seed in SEEDS and frames==120,'Unadmitted accuracy controls.') + tees={} + def capture(argv,source): + method=argv[2] + require(method in CAPTURE_METHODS and int(argv[3])==seed and method not in tees,'Unexpected accuracy launch.') + tee=TeeReader(source,wire_dir/f'{method}-{seed}.bin');tees[method]=tee;return tee + transport.capture_hook=capture + try:returned=original_compare(n,seed,frames,model) + finally:transport.capture_hook=None + result,ref,predicted,teacher,learned=returned + require(set(tees)==set(CAPTURE_METHODS),'Missing captured accuracy process.') + for method,states,reports in [('jacobi',teacher,ref['reports']),('learned',learned,predicted['reports'])]: + indices.append(retain_trajectory(tees[method],method,seed,states,reports,output,identity)) + require(result.get('max_ai_divergence')==max(r['actual_divergence_max'] for r in predicted['reports']), + 'Accuracy divergence maximum differs.') + progress('accuracy',len(indices)//2,4,max_relative_velocity_l2=result['max_relative_velocity_l2'], + max_relative_tracer_l2=result['max_relative_tracer_l2']) + return returned + def rollout(n,method,seed,frames,model=None,capture=False): + nonlocal timed + returned=original_rollout(n,method,seed,frames,model,capture=capture) + if not capture: + timed+=1;progress('benchmark',timed,60) + return returned + exp.compare=compare;exp.rollout=rollout + progress('accuracy',0,4) + from types import SimpleNamespace + exp.evaluate(SimpleNamespace(output=run_dir)) + source=run_dir/'evaluation' + result_raw=generate.capture(source/'result.json',32*1024*1024)[0];result=generate.decode(result_raw) + validate_result(result,freeze,generate.sha(raws['freeze'])) + require(len(transport.evidence)==68 and timed==60 and len(indices)==8,'Incomplete native execution inventory.') + original_names={f'{m}-{s}.npz' for m in CAPTURE_METHODS for s in SEEDS}|{'viewport.json'} + require(set(result.get('artifact_sha256',{}))==original_names,'Original evaluation artifact inventory differs.') + for name in original_names:require(file_hash(source/name)==result['artifact_sha256'][name],'Original evaluation output hash differs.') + progress('publishing',0,1) + for name in original_names|{'result.json'}: + with (source/name).open('rb') as read,(output/name).open('xb') as write: + shutil.copyfileobj(read,write,1024*1024);write.flush();os.fsync(write.fileno()) + require(file_hash(source/name)==file_hash(output/name),'Evaluation output copy differs.') + retained={'input-model.pt':raws['checkpoint'],'input-freeze.json':raws['freeze'], + 'input-training-receipt.json':raws['training_inputs'],'input-training-result.json':raws['training_result'], + 'input-build-receipt.json':raws['build_receipt']} + for name,raw in retained.items(): + with (output/name).open('xb') as handle:handle.write(raw);handle.flush();os.fsync(handle.fileno()) + summary={'format':'tuldok_result_summary_v1','title':'Autonomous CPU evaluation', + 'metrics':[{'name':'Timed frames','value':7200,'unit':'frames'}, + {'name':'Accuracy frames','value':960,'unit':'frames'}, + {'name':'Speedup vs Jacobi','value':result['speedup_vs_jacobi'],'unit':'ratio'}, + {'name':'Maximum velocity relative L2','value':max(v['max_relative_velocity_l2'] for v in result['accuracy']),'unit':''}], + 'scope':['Fresh CPU checkpoint; previously exposed seeds900..903.','Five repetitions; original numerical algorithms.', + 'All six fields and exact accuracy stdout retained.','Evaluated input checkpoint; no retraining or GPU acceptance.'], + 'checkpoint':'input-model.pt'} + (output/'result-summary.json').write_bytes(encode(summary)) + inventory={name:{'bytes':(output/name).stat().st_size,'sha256':file_hash(output/name),'role':role} + for name,role,_cap in outputs() if name!='evaluation-receipt.json'} + receipt={'schema_version':1,'evaluation':'fresh_cpu_autonomous_evaluation','identity':identity,'input_evidence':evidence, + 'native_policy':generate.decode(policy),'native_launches':transport.evidence,'outputs':inventory, + 'limitations':['Previously exposed seeds; no independent holdout or GPU claim.', + 'Explicit provenance, transport and retention hooks around original numerical evaluate.', + 'Wire evidence is native stdout; pressure input bytes are not retained.']} + (output/'evaluation-receipt.json').write_bytes(encode(receipt)) + total=0 + for name,_role,cap in outputs(): + size=(output/name).stat().st_size;require(size<=cap,'Declared evaluation file limit exceeded.');total+=size + require(total<=TOTAL_LIMIT,'Aggregate evaluation output bound exceeded.') + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + for name in ('checkpoint','freeze','training-inputs','training-result','build-receipt','input-evidence', + 'native-policy','runtime-snapshot-sha256','output'):parser.add_argument('--'+name,required=True) + run(parser.parse_args()) + + +if __name__=='__main__':main() diff --git a/adapters/rheon/evaluation_project.py b/adapters/rheon/evaluation_project.py new file mode 100644 index 00000000..1091c24e --- /dev/null +++ b/adapters/rheon/evaluation_project.py @@ -0,0 +1,74 @@ +"""Assemble evaluation sources; inputs are selected from one completed training Run.""" +import argparse +import os +from pathlib import Path +import shutil +import uuid + +if __package__: + from . import evaluate, cpu_compare, generate +else: + import evaluate + import cpu_compare + import generate + + +def definition(sources,snapshot): + evaluate.require(type(snapshot) is str and generate.HASH.fullmatch(snapshot),'Runtime snapshot SHA256 required.') + native='lab/target/release/rheon-trajectory-parallel' + arguments=[] + for slot in evaluate.INPUTS:arguments.extend([{'literal':'--'+slot.replace('_','-')},{'input':slot}]) + for name,token in [('build-receipt',{'input':'build_receipt'}),('native-policy',{'native_policy':native}), + ('input-evidence',{'input_evidence':True}),('runtime-snapshot-sha256',{'literal':snapshot}), + ('output',{'output_dir':True})]:arguments.extend([{'literal':'--'+name},token]) + return {'schema_version':1,'format':'external_recipe_v1','id':'rheon-autonomous-evaluation', + 'name':'Rheon autonomous CPU evaluation','version':1,'runtime':'captured_cpu_python_v1', + 'runtime_snapshot_sha256':snapshot,'sources':[{'path':name,'sha256':generate.sha(raw)} for name,raw in sorted(sources.items())], + 'native_sources':[native],'entrypoint':'evaluate.py','arguments':arguments,'parameters':{}, + 'inputs':{**{name:{'max_bytes':cap} for name,(_role,cap) in evaluate.INPUTS.items()},'build_receipt':{'max_bytes':1024*1024}}, + 'outputs':[{'path':name,'role':role,'max_bytes':cap,'required':True} for name,role,cap in evaluate.outputs()], + 'environment':{},'resources':{'profile':'captured_cpu_large_outputs_v1','wall_seconds':2400,'max_output_bytes':evaluate.TOTAL_LIMIT}} + + +def assemble(lab_root,binary,build_receipt,runtime_snapshot_sha256,destination): + lab=Path(lab_root);adapter=Path(__file__).parent + native=generate.capture(binary,8*1024*1024)[0];build=generate.capture(build_receipt,1024*1024)[0] + evaluate.require(generate.sha(native)==cpu_compare.BINARY_SHA256 and generate.sha(build)==cpu_compare.BUILD_RECEIPT_SHA256, + 'Qualified parallel build required.') + sources={'lab/target/release/rheon-trajectory-parallel':native} + for name,digest in cpu_compare.SOURCES.items(): + raw=generate.capture(lab/'scripts'/name,65536)[0] + evaluate.require(generate.sha(raw)==digest,'Original numerical source differs.') + sources['lab/scripts/'+name]=raw + for name in ('evaluate.py','cpu_compare.py','generate.py','native_transport.py'): + sources[name]=generate.capture(adapter/name,256*1024)[0] + recipe=generate.encode(definition(sources,runtime_snapshot_sha256)) + manifest={'schema_version':1,'id':uuid.uuid4().hex,'name':'Rheon autonomous CPU evaluation', + 'data':[{'id':'build_receipt','label':'Parallel build evidence','path':'build-receipt.json','sha256':generate.sha(build)}], + 'recipes':[{'id':'evaluate','label':'Evaluate completed training checkpoint','path':'recipe.json','sha256':generate.sha(recipe)}], + 'evidence':[],'requirements':[{'id':'completed-training','label':'Bind four verified artifacts from one completed training Run'}]} + files={**sources,'build-receipt.json':build,'recipe.json':recipe,'project.json':generate.encode(manifest)} + destination=Path(destination);destination.mkdir(mode=0o700,exist_ok=False) + try: + for name,raw in files.items(): + path=destination/name;path.parent.mkdir(parents=True,exist_ok=True) + with path.open('xb') as handle: + handle.write(raw);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno()) + for directory in [*sorted((p for p in destination.rglob('*') if p.is_dir()),reverse=True),destination]: + fd=os.open(directory,os.O_RDONLY|os.O_DIRECTORY) + try:os.fsync(fd) + finally:os.close(fd) + except BaseException: + shutil.rmtree(destination);raise + return manifest + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + for name in ('lab-root','binary','build-receipt','runtime-snapshot-sha256','destination'):parser.add_argument('--'+name,required=True) + args=parser.parse_args() + manifest=assemble(args.lab_root,args.binary,args.build_receipt,args.runtime_snapshot_sha256,args.destination) + print(generate.encode({'project_id':manifest['id'],'manifest':str(Path(args.destination)/'project.json')}).decode()) + + +if __name__=='__main__':main() diff --git a/adapters/rheon/native_transport.py b/adapters/rheon/native_transport.py index 2682881e..caa8e40e 100644 --- a/adapters/rheon/native_transport.py +++ b/adapters/rheon/native_transport.py @@ -42,11 +42,17 @@ def exact(stream, size): class Transport: PIPE = subprocess.PIPE - def __init__(self, binary, policy_raw, *, allowed_seeds=(900, 901, 902, 903), capture_diagnostics=False): + def __init__(self, binary, policy_raw, *, allowed_seeds=(900, 901, 902, 903), capture_diagnostics=False, evaluation_mode=False, capture_hook=None): if type(allowed_seeds) is not tuple or allowed_seeds not in ((900, 901, 902, 903), (500, 501)): raise ValueError('Unsupported reviewed native seed scope.') if type(capture_diagnostics) is not bool: raise ValueError('Invalid diagnostic capture flag.') + if type(evaluation_mode) is not bool or (capture_hook is not None and not callable(capture_hook)): + raise ValueError('Invalid evaluation capture hook.') + if (evaluation_mode and allowed_seeds != (900, 901, 902, 903)) or (capture_hook is not None and not evaluation_mode): + raise ValueError('Evaluation capture requires the closed evaluation seed scope.') + self.evaluation_mode = evaluation_mode + self.capture_hook = capture_hook self.capture_diagnostics = capture_diagnostics self.diagnostics = [] self.allowed_seeds = tuple(str(seed) for seed in allowed_seeds) @@ -110,10 +116,12 @@ def wait_with_diagnostics(*wait_args, **wait_kwargs): or ready.get('face_lengths') != [33*32*32]*3): raise ValueError('Native READY differs from requested controls.') maps = generate.mapping_evidence(process.pid, self.observed, native_fd, loader_fd) - if len(self.evidence) >= 64: + if len(self.evidence) >= (72 if getattr(self, 'evaluation_mode', False) else 64): raise ValueError('Native launch count exceeds admitted comparison.') self.evidence.append({'method': args[2], 'seed': int(args[3]), 'ready': ready, 'mappings': maps}) process.stdout = PrefixReader(header + raw, process.stdout) + if getattr(self, 'capture_hook', None) is not None: + process.stdout = self.capture_hook(args, process.stdout) return process except BaseException: if process is not None: diff --git a/app.py b/app.py index 18fd7864..d749b2b5 100644 --- a/app.py +++ b/app.py @@ -440,6 +440,24 @@ def reply(self, data, status=200, content_type='application/json', headers=()): except (BrokenPipeError, ConnectionResetError): pass + def reply_stream(self, stream, metadata): + # The owner has already verified a private spool before HTTP success. + self.send_response(200) + self.send_header('Content-Type', metadata['media_type']) + self.send_header('Content-Length', str(metadata['bytes'])) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.end_headers() + try: + while True: + raw = stream.read(65536) + if not raw: + break + self.wfile.write(raw) + except (OSError, ConnectionError): + # Headers are committed; never append a JSON error to payload. + self.close_connection = True + def project_access(self): # This opt-in file inspection surface is local-only until actor authentication. host = urlsplit('http://' + self.headers.get('Host', '')).hostname @@ -578,6 +596,8 @@ def _get(self): return self.reply(dataset.pipelines.list()) if len(parts) == 2 and parts[0] == 'graphs': return self.reply(dataset.pipelines.get(parts[1])) + if parts == ['run-artifact-inputs']: + return self.reply(dataset.runs.input_candidates()) if parts == ['runs']: return self.reply(dataset.runs.list()) if len(parts) == 2 and parts[0] == 'runs': @@ -588,8 +608,8 @@ def _get(self): query = parse_qs(urlsplit(self.path).query) return self.reply(dataset.runs.predictions(parts[1], query.get('split', ['validation'])[-1])) if len(parts) == 4 and parts[0] == 'runs' and parts[2] == 'artifacts': - data, media_type = dataset.runs.artifact(parts[1], parts[3]) - return self.reply(data, content_type=media_type) + with dataset.runs.open_artifact(parts[1], parts[3]) as (stream, metadata): + return self.reply_stream(stream, metadata) return self.reply({'error': 'Workspace route not found.', 'code': 'not_found'}, 404) if path == '/api/workbench/releases/input-candidates': return self.reply(dataset.releases.input_candidates()) diff --git a/docs/contracts/captured-cpu-runtime.md b/docs/contracts/captured-cpu-runtime.md index 1fe9bc80..b5d3ecd0 100644 --- a/docs/contracts/captured-cpu-runtime.md +++ b/docs/contracts/captured-cpu-runtime.md @@ -110,3 +110,15 @@ original descriptor and specification after relocation; new execution resolves t current snapshot root and requires fresh approval. Hash custody is not scientific acceptance. Actual CPU computation, checkpoint compatibility, numerical gates and UI workflows require their own qualification evidence. + +## Optional large-output resource profile + +`resources.profile = "captured_cpu_large_outputs_v1"` selects bounded large +outputs only with the existing qualified captured CPU runtime. Helper validation +checks the complete descriptor/bootstrap before admitting 2400 seconds and +4 GiB aggregate outputs (at most 96 files, each at most 512 MiB). Default CPU and +system profiles retain their old shapes, limits and historical identities. +This extension does not enlarge captured input/source or diagnostic streams, +alter cancellation/drain deadlines, enable network isolation, or qualify GPU/JIT. +Actual recipes retain explicit smaller output budgets and require measured +capacity admission. See the external recipe contract for custody and downloads. diff --git a/docs/contracts/external-recipes.md b/docs/contracts/external-recipes.md index 4b078ce8..0b96f97a 100644 --- a/docs/contracts/external-recipes.md +++ b/docs/contracts/external-recipes.md @@ -227,3 +227,50 @@ artifacts remain readable even when separate staging diagnostics are unavailable The UI labels metrics as recipe-reported, keeps logs/provenance in details and uses the existing artifact download route. Neither stdout nor a filename suffix can nominate a result or enable a checkpoint download. + +## Completed-Run inputs and optional large CPU outputs + +A new exact input binding is `{kind:"run_artifact", run_id, artifact_name, +expected_sha256}`. The existing Runs owner captures only a completed, drained, +same-catalog external artifact, verifies custody and payload bytes, and enforces +the declared input limit before reading. Failed, cancelled or unknown Runs are +ineligible. Its material receipt is `{kind:"run_artifact", source:}`; old project and frozen-release receipts are unchanged. +Cold validation reuses the custody owner's source validator without accessing +the original catalog. It checks internal retained relationships, not historical +database authenticity. Editing a binding invalidates material approval. + +The optional argument token `{"input_evidence":true}` resolves to captured +`runtime-bootstrap/input-evidence.json`: canonical JSON mapping input slot names +to their exact typed material receipts. The generated bytes are hashed in capture +and recomputed during cold decoding. It carries no credentials or caller-authored +provenance. Definitions without this token gain no capture or identity field. + +The explicit optional `resources.profile` value +`captured_cpu_large_outputs_v1` requires runtime `captured_cpu_python_v1`. Its +limits are 2400 wall seconds, 96 flat declared outputs, 512 MiB per file and 4 GiB +aggregate. Definitions still declare smaller per-file/aggregate limits as needed. +The helper's matching `resource_profile` field is accepted only with the verified +CPU descriptor/bootstrap. Old omitted-profile shapes and bounds are unchanged. +Input and source capture limits remain independent and unchanged from P5b. + +Output verification and publication hash/copy bounded chunks and preserve the +existing manifest, receipt, fsync and completed-state sequence. Publication does +not rename or delete producer staging. Large downloads use one private anonymous +spool, fully verified before HTTP 200, then bounded transfer; failures/disconnects +close it. Current disk capacity is checked before large admission, publication +and spooling; this is not a filesystem reservation against concurrent writers. + +For large-profile Runs only, list/get exposes receipt-verified inventory with +payloads explicitly not checked. Individual downloads/captures verify selected +bytes; full audit verifies every artifact. Small-profile reads keep their existing +whole-inventory verification semantics. A result summary verifies its own bytes +and referenced checkpoint, without certifying unrelated large field artifacts. +The UI labels recorded outputs as verified on download. + +Inspect-authorized `GET /api/workspace/run-artifact-inputs` returns at most 100 +completed external Runs with candidate artifacts at most 40 MiB. Receipt-verified +metadata is a selection hint; capture remains the byte authority. Closed Run rows +contain run_id, state, status, reason and artifacts. Available artifact rows contain +artifact_name, role, bytes, sha256 and status. Unavailable rows disclose no untrusted +artifact names. No classification holdout or arbitrary filesystem path is exposed. diff --git a/docs/contracts/rheon-evaluation.md b/docs/contracts/rheon-evaluation.md new file mode 100644 index 00000000..f76e77a0 --- /dev/null +++ b/docs/contracts/rheon-evaluation.md @@ -0,0 +1,129 @@ +# Optional autonomous CPU evaluation + +This recipe consumes the checkpoint and immutable fresh freeze from one completed +training Run. It preserves original `trajectory_experiment.evaluate` numerical +algorithms, five benchmark repetitions, seeds900–903 and120frames. Explicit +provenance, native transport, retention and progress hooks are orchestration +changes; it is not presented as an untouched original CLI. Original source bytes +are pinned and retained. No training, model installation or GPU execution occurs. + +## Project and direct inputs + +`evaluation_project.assemble(lab_root, binary, build_receipt, +runtime_snapshot_sha256, destination)` creates a new project without execution; +CLI flags are `--lab-root`, `--binary`, `--build-receipt`, +`--runtime-snapshot-sha256`, `--destination`. The project reference `evaluate` +selects definition `rheon-autonomous-evaluation`. Original four lab scripts and +parallel native retain their captured layout. Assembly refuses overwrite. + +The graph must bind four completed-Run inputs: `checkpoint` (checkpoint role), +`freeze` (training_freeze), `training_inputs` (training_inputs) and +`training_result` (training_result). `build_receipt` is the qualified static build +project reference. Generic core captures a canonical slot→typed receipt object +through `{'input_evidence': true}`; no caller-supplied provenance or manual +checkpoint-copy bridge substitutes for this path. The wrapper checks four input +roles, measured hashes/sizes, same catalog/Run/attempt/material/custody identity, +and mutual checkpoint/freeze/training-array/release/inputreceipt payload hashes. + +The wrapper accepts `--checkpoint`, `--freeze`, `--training-inputs`, +`--training-result`, `--build-receipt`, `--input-evidence`, `--native-policy`, +`--runtime-snapshot-sha256`, `--output`. Grid, methods, seeds, frame count, +repetitions and thread policy come from the verified freeze, not adjustable +runtime parameters. CPU snapshot/Torch/NumPy/native/build/original-source identity +must match. CPU weights_only checkpoint loading checks original schema, full +positive finitefloat64 gains32767 and basis32×32. + +## Numerical and custody hooks + +The original evaluate entry normally obtains provenance from a live Git checkout. +This wrapper instead verifies captured original script/native/build identities and +actual framework/snapshot version, then provides that verified fresh provenance +through the module-local hook. It does not invent historical Rust/Git provenance +or rewrite the original freeze. Original model loading, comparisons, warmup, +rotated benchmark loops, output encoding and metrics remain unchanged. + +The shared transport admits a closed evaluation mode with at most72 launches; +this recipe requires exactly68 (eight accuracy and60timed). Other recipes keep +their64-launch default. Each launch uses sealed native/loader descriptors, fixed +loader policy, READY controls and observed mapping identity. Native stdout is +teed only for accuracy captures. Each task-owned tee is bounded to170MiB; +no more than the current reference/learned pair is retained. Original close waits +for native exit before the tee checks EOF, so trailing stdout refuses completion. +Failed attempts retain their task-private wire/output evidence under the existing +attempt directory. The existing Runs helper owns process descendants, deadlines +and cancellation. + +A compare wrapper calls the original compare function and retains its returned +arrays alongside exact wire. Learned native emits an extra f64 RHS request before +each report; that request must equal the subsequently retained RHS field. Six +NPY payloads must exactly equal original returned array bytes, preserving dtype, +MAC placement, CZYX shape and stage. Native reports and returned reports agree. +Accepted fields/reports must be finite, tracer within[0,1], native residual≤1e-8, +divergence≤1e-5,Courant≤1, and learned corrections zero. Original velocity/tracer +relativeL2 errors must be finite and strictly below1e-5; maxima, benchmark coverage, +rotation, medians and ratios are checked. Centered-pressure error and tracerRMSE +remain original finite metrics. Peak inferenceRSS must stay below8GB with zeroGPU +allocation. Physical units remain native/unspecified; stages are not simultaneous. + +## Retention and output schema + +The captured CPU recipe selects `resources.profile=captured_cpu_large_outputs_v1`, +wall2400seconds and aggregate3,500,000,000bytes. It declares89 required flat files: + +| Artifacts | Count | Per-file cap | +| --- | ---: | ---: | +| {jacobi,learned}-{900..903}.npz |8|128MiB| +| fields-{method}-{seed}-{000..007}.zip |64|40MiB| +| wire-{method}-{seed}.json |8|256KiB| +| viewport.json |1|320MiB| +| result.json |1|32MiB| +| evaluation-receipt.json |1|8MiB| +| input-model.pt |1|1MiB| +| input-freeze.json |1|2MiB| +| input-training-receipt.json |1|2MiB| +| input-training-result.json |1|4MiB| +| input-build-receipt.json |1|1MiB| +| result-summary.json |1|64KiB| + +Original NPZ, viewport and result bytes are preserved. Original result.json hashes +only its original eightNPZ and viewport, as before; supplementary chunks are +outside that directory while the original entry computes hashes. The outer +receipt binds all added outputs and original outputs. NPZ omits RHS; supplemental +scientific chunks retain allsix fields. No duplicate Data/release import of all +chunks is required; representative inspection is separate from complete retained +Run evidence. + +Each ZIP is standard scientific_field_sequence_v1 with16 consecutive frames, +except the final8. `arrays/{absolute_frame:03d}-{field}.npy` retains each field. +`sources/wire.bin` is one disjoint original stdout range; the first includes READY. +`sources/evaluation-trace.json` contains schema_version, method, seed, READY, +frame_start/end, stdout_start/end, stdout_sha256, includes_ready, original reports, +learned_rhs_request_before_report and captured identity. Family follows the +original scene/grid/seed convention, shared acrossmethods/chunks. Time is absolute +native frame time, not reset perchunk. + +Each wire index contains schema_version, method, seed, stdout_bytes/hash, frames120, +identity and ordered chunks with path/hash/bytes, frame_start/end and exact stdout +range/hash. Ordered ranges reconstruct complete native stdout, including learned +RHS requests. Every written chunk member and final index is reread/verified before +removing only its task-created redundant tee. Native pressure input bytes are not +claimed retained; these are exact stdout records, not bidirectional capture. + +The evaluation receipt contains schema_version, evaluation label, identity, +input_evidence, native_policy, all68 native launch/mapping entries, output inventory +and limitations. Identity binds checkpoint/freeze/runtime/native/original-source +hashes and shared originating training authority. Output inventory excludes the +receipt itself and binds every other artifact's bytes/hash/role. Core's artifact +manifest independently binds the receipt. Result summary points to same-run +`input-model.pt` as the evaluated input checkpoint, not a newly trained output. + +## Scope and acceptance + +Live progress reports completed accuracy seeds and timed trajectories only; +publication is an existing Runs state transition. No ETA or progress inferred +from elapsed wall time. Previously exposed seeds remain explicitly disclosed. +Source/unit tests prove boundaries only. Actual autonomous evaluation requires +prior P5b gates, independent source review, measured disk admission, completed +Run custody, independent wire/NPY/NPZ/viewport and numerical verification, and +real train→evaluate UI/restart evidence. No historical timing, CUDA/Pumas or +unseen-holdout acceptance follows from this contract. diff --git a/docs/plans/generic-scientific-projects/plan.md b/docs/plans/generic-scientific-projects/plan.md index 3ecaa526..61af851d 100644 --- a/docs/plans/generic-scientific-projects/plan.md +++ b/docs/plans/generic-scientific-projects/plan.md @@ -2,9 +2,9 @@ **Plan status:** `Active` -**Current phase:** P5a qualified at draft PR50 commit05ea66e; P5b actual frozen-input CPU fit qualified locally; final regression and draft publication in progress. +**Current phase:** P5a qualified at draft PR50 commit05ea66e; P5b published at draft PR51 commit28c666f; isolated P5c source preparation admitted while final P5b gates run. -**Next slice:** Finish P5b regression/publication, then admit P5c autonomous evaluation with completed-Run artifact inputs and durable full fields. See [P5b admission](reports/p5b-admission.md). +**Next slice:** Implement P5c direct Run-artifact inputs, bounded large-output execution and original autonomous evaluation; no real evaluation before all dependency and review gates pass. See [P5c admission](reports/p5c-admission.md). **Acceptance status:** `partial` diff --git a/docs/plans/generic-scientific-projects/reports/p5c-admission.md b/docs/plans/generic-scientific-projects/reports/p5c-admission.md new file mode 100644 index 00000000..74b33ac3 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5c-admission.md @@ -0,0 +1,485 @@ +# P5c admission: autonomous evaluation and durable full fields + +## Admitted source preparation and dependency gates + +P5b actual fitting, independent NumPy/checkpoint audit, both validation trajectories, +positive physics ablation, browser progress/download and cold restart passed. It +is published as draft PR51 at commit28c666f. Full final regression and hosted CI +are still pending. Admit source preparation and bounded authored tests ONLY in +`/workspace/Tuldok-p5c`, branch `feature/scientific-autonomous-evaluation-20261013`; +P5b remains source-frozen in its own worktree. Actual evaluation waits for P5b's +remaining gates, independent P5c source review and measured capacity admission. +This sequencing amendment permits useful isolated work without claiming any +pending dependency accepted. No merge, deployment or model installation. + +The source/numerical proposal below is admitted with these authoritative decisions: + +- Keep the original numerical algorithms, five benchmark repetitions, four + previously exposed test seeds and immutable fresh checkpoint/freeze. +- New optional `resources.profile = "captured_cpu_large_outputs_v1"` is the closed + resource variant. It requires the existing captured CPU runtime. Old definitions + omit this field and retain identical material/spec shapes and limits. The helper + receives the exact same optional `resource_profile` variant, validated against + a fully verified CPU descriptor; a number alone cannot enlarge old limits. +- Large bounds: 2400 seconds, 96 flat outputs, 512 MiB each, 4 GiB aggregate maximum. + This actual evaluation declares aggregate output 3,500,000,000 bytes. Keep + capture/input bounds from P5b unchanged, plus cancellation/drain semantics. +- Use bounded streaming-copy publication and a single verified anonymous download + spool. No publication rename or per-file staging deletion. Preserve existing + manifest/receipt encoding and recovery ownership. +- Add exact completed-Run artifact inputs through the existing Runs owner. For + adapters needing origin evidence, add optional typed `{"input_evidence": true}` argv token: + it resolves to a captured canonical JSON object mapping input slot names to + their existing typed material receipts. Its bytes are derived from the admitted + receipts, hashed as a capture and checked again during cold material decoding. + It contains no caller-supplied provenance or credentials. Definitions not using + the token gain no new capture or identity field. +- The optional evaluator checks all four training artifact inputs have the same + completed Run/receipt identity, exact roles and mutually agreeing payload hashes + before loading the checkpoint. Core code contains no Rheon semantics. +- Retain all six fields and exact native stdout for every accuracy trajectory in + bounded chunks. Tee temporaries are bounded per trajectory and removed only + after verified complete chunk retention; do not accumulate eight duplicate wires. + Preserve original NPZ/report/viewport bytes and declare added orchestration. +- Recheck disk before real admission. The verified task-owned installer cleanup + reclaimed 1,685,254,144 bytes; it did not remove scientific evidence, toolchain, + sources or binaries. Actual free space and temporary peak must justify the + two-copy output budget plus spool/headroom. No additional cleanup is presumed. + +Write owners: root owns recipe_contract.py, pipeline.py, narrow access whitelist +if required, new input/profile tests and plan/contracts. volume_assets owns +runs.py/app.py and external artifact/progress/HTTP tests. milestone_audit owns +recipe_process.py and its focused tests. recipe_design owns optional evaluation +adapter/project/native transport changes and numerical unit tests. mesh_ui owns +workspace-external.js/workspace-real.js/controller tests. point_assets owns +independent numerical oracle and real evaluation browser harness. mesh_review +finishes P5b regression, then performs independent P5c review. No owner edits a +shared file without coordination. Heavy real workloads remain gated. + +## Reviewed numerical and generic design record + + +This proposal preserves the reviewed original numerical algorithms while making +orchestration changes explicit. No model execution, source changes, package +installation or evaluation qualification was performed for this proposal. +P5b must first produce a real new CPU checkpoint, its training/validation evidence, +and a matching immutable freeze bound to reviewed/frozen training data. + +## Exact original entry and dependency behavior + +Pinned lab `scripts/trajectory_experiment.py:evaluate(args)` reads model.pt and +freeze.json from args.output, verifies checkpoint SHA, calls provenance(), and +requires binary_sha256, source_sha256 and Torch version equal the freeze. It then +creates a new evaluation/ directory. It loads ModalPressure on freeze.device, +compares four seeds 900–903 over120 frames, writes eight compressed NPZ files and +one first-seed viewport.json, performs20 resident-model warmups, then executes +freeze.benchmark_repetitions rotated loops of Jacobi/SGS/learned. Original train() +sets benchmark_repetitions=5; the default main() sets Torch threads1. + +The original compare() preserves complete teacher/predicted state arrays. It +computes full velocity/tracer relativeL2, tracerRMSE and mean-centered pressure +relativeL2 per frame, plus maxima and learned divergence. Unlike P5a's separate +verify_parallel_results.metrics(), this original comparison does include a +pressure metric. RHS has no original accuracy score; retaining six fields must +not be described as the original algorithm scoring all six independently. + +Original NPZ keys: tracer, pressure, velocity_0, velocity_1, velocity_2. Each value +stacks all120 original flattened frame arrays. RHS is omitted, so NPZ alone does +not satisfy durable all-six-field retention. viewport.json preserves fullfloat32 +tracers converted to JSON numbers for first-seed reference/learned frames and +original reports/accuracy; it is not a downsampled preview. result.json binds +freeze/checkpoint, all accuracy records, benchmark records, medians, ratios, +timing scope, warmup/memory, and hashes of existing evaluation files. + +provenance() reads original experiment/model, all Rheon Rust files, trajectory +Rust source/manifest/lock, and runs git rev-parse on Rheon. Captured source-only +projects have no Git checkout. Do not fabricate Git metadata, replace recorded +source identities with wrapper identities, or call this an untouched CLI entry. + +## Recommended optional adapter architecture + +Add adapters/rheon/evaluate.py + evaluation_project.py and focusedtests/contract; +reuse qualified captured CPU runtime and sealed native transport. Retain exact +original script/model/native/source inventory and immutable input bytes. Wrapper +sets explicit admitted CPU/native threads before loading model and records them; +choose the P5b-frozen thread policy, not parameters tuned on evaluation results. + +Invoke original evaluate(args) after installing THREE narrowly declared hooks: + +1. Module-local subprocess launch proxy for sealed native/loader, mapped identity + and exact wire custody. No global subprocess monkeypatch or second scheduler. +2. Module-local provenance() adapter: recompute all captured original source and + native hashes and actual runtime version, compare them with P5b-frozen receipt, + then return that verified provenance. The original entry's binary/source/Torch + equality checks still run. Original provenance Git execution is explicitly + replaced by captured, independently verified source/build receipt identity. +3. A compare() wrapper calls the ORIGINAL compare function unchanged, retains its + returned reports/arrays into scientific chunks, and returns the original + tuple unchanged. No numerical conversion, norm replacement or model changes. + +Capture byte-exact native stdout per accuracy trajectory. The proxy tees reads +only when capture commands (02) are used; benchmark trajectories (01) must not +incur full-field retention. READY and native JSON/frame bytes remain original. +Learned wire includes an additional f64 RHS request before the accepted-frame +report/arrays. Never treat it as the sole post-frame RHS or silently discard it. +Record exact offsets/type/shape/stage and retain those extra received bytes. +If pressure input bytes are retained, put them in a separate bounded artifact +rather than overfilling a 40MiB scientific source ZIP. Native input commands are +recorded as control metadata; do not claim retained stdout is bidirectional wire. + +Stream raw byte tee to task-owned files, generate ZIP_STORED scientific bundles +in ≤16-frame chunks, release temporary references after durable closed chunk +creation. Original compare still holds its current teacher/predicted arrays as +required by the original algorithm; do not call it streaming memory acceptance. +The wrapper creates chunk outputs OUTSIDE original evaluation/ until evaluate() +hashes its original NPZ/viewport and emits result.json. Thus the original result's +artifact hash map remains original output semantics. An additional outer receipt +binds ALL original and supplementary output artifacts together. + +The wrapper invocation and documentation must say 'original evaluate numerical +algorithms with explicit provenance/transport/retention orchestration', not +'untouched evaluate CLI'. No source file bytes are edited to install these hooks. + +## Inputs and binding to actual completed work + +Direct completed-Run artifact binding is REQUIRED for the in-app train→evaluate +path. No manual download/project-data bridge is an acceptance substitute. +Extend the existing input union with the closed shape: +`{kind:"run_artifact",run_id:<32hex>,artifact_name:,expected_sha256:<64hex>}`. +Keep old project_reference and frozen_release encodings unchanged. + +Use existing inspect-authorized Runs.capture_artifact(run_id,name,expected_sha256) +as the custody authority, with a declared input max-byte bound applied BEFORE +reading bytes. It already verifies published artifact inventory, exact SHA, +custody receipt identity/hash, completed outcome and drained state, and returns +captured bytes plus server-derived source evidence. Add only the missing generic +input adapter/candidate projection; do not introduce another artifact catalog. +The candidate picker exposes completed same-catalog run/artifact name/role/hash/ +bytes, bounded by existing list limits. Editing any selection invalidates prior +preflight/approval. Revalidate every selected artifact during material capture +before GO; failed/cancelled/unknown/missing/tampered/wrong-catalog inputs fail. + +Exact evaluation input slots: +- checkpoint ← model.pt, declared role checkpoint, ≤1MiB; +- freeze ← freeze.json, role training_freeze, ≤2MiB; +- training_inputs ← input-receipt.json, role training_inputs, ≤2MiB; +- training_result ← result.json, role training_result, ≤4MiB; +- build_receipt ← existing static qualified build project reference, ≤1MiB. + +The optional adapter requires the four artifact receipts identify the SAME +completed P5b Run and binds all measured hashes. Generic core validates custody +and roles; it has no Rheon-specific artifact names. The complete server-derived +input receipt, including originating run/attempt/material/custody identity, must +be captured as a declared generated input-evidence file/token so the wrapper can +verify common origin; do not pass only raw bytes and lose run authority. Existing +material input receipts remain canonical authority; generated evidence is their +hash-bound projection, not a new authority. A `{input_receipt:}` typed token +is one narrow option, with fixed generated path and retained validate_material +checks matching the exact receipt. Captured receipt totals count toward bounds. + +Do not recapture training.npz or six training archives just to evaluate: the model +consumes its checkpoint; training_result/inputreceipt/freeze bind real prior +training NPZ/release hashes and artifact custody. Independent training acceptance +already verifies actual training-array use. Release IDs remain navigable evidence. + +Reject mismatched checkpoint/freeze/runtime/native/source/training release +identities before launch. Do not rewrite historical freeze or adjust P5b's +immutable freeze after seeing test behavior. If P5b's original freeze saysfive +benchmark repetitions, executefive. Any newly chosen lower-count qualification +must be a separately named and approved new evaluation configuration fixed before +test generation, with original freeze retained; it cannot masquerade as the +original five-repeat evaluation. + +## Minimum generic execution/output changes + +Current external recipe output limits are32 flat files,64MiB per-file and total; +verify_outputs() additionally retains each whole file in memory while hashing. +These do not admit the real original eight NPZ/viewport or full-field chunks. +Recommended finite optional server-approved large-output resource profile: + +- At most96 explicitly declared flat files; no directory/glob discovery contract. +- At most512MiB per file, at most4GiB total output; exact lower per-file declarations. +- Preserve existing small-profile defaults and immutable admission/material digest. +- Chunked descriptor hashing with bounded buffers and existing before/after file + identity checks; publication likewise streams bounded copies+hash checks. +- Same approved definition, sole Runs queue/helper, output verification, restart + fencing and artifact manifest. No plugin framework, separate runner or side DB. +- Scientific ZIP limit remains40MiB; increasing generic artifacts does not enlarge + the scientific parser's per-asset contract. + +Exact deterministic declaration inventory (89 flat files, all required): + +| Names | Count | Per-file bound | Role | +| --- | ---: | ---: | --- | +| {jacobi,learned}-{900,901,902,903}.npz |8|128MiB|evaluation_arrays| +| fields-{method}-{seed}-{000..007}.zip |64|40MiB|scientific_fields| +| wire-{method}-{seed}.json |8|256KiB|native_wire_index| +| viewport.json |1|320MiB|evaluation_viewport| +| result.json |1|32MiB|evaluation_result| +| evaluation-receipt.json |1|8MiB|evaluation_receipt| +| input-model.pt |1|1MiB|checkpoint| +| input-freeze.json |1|2MiB|source_freeze| +| input-training-receipt.json |1|2MiB|training_inputs| +| input-training-result.json |1|4MiB|training_result| +| input-build-receipt.json |1|1MiB|source_build| +| result-summary.json |1|64KiB|result_summary| + +Methods are exactlyJacobi/learned and seeds900..903. Eight chunks are frames +1–16,17–32,…,97–112,113–120. The finalchunk is8frames; no padding or duplicate +frames. Per-file maxima sum below4GiB, with total4GiB enforced independently. +Keep original output filenames and bytes for eightNPZ,viewport,result. The generic +role scientific_fields enables existing importUI; summary checkpoint references +same-run retained input-model.pt with checkpointrole. State it is evaluated input, +not a newly trained output. + +Each chunk retains its exact stdout range and sixNPY arrays. Include absolute +frame/time range and stream byte offsets. Complete stdout reconstructs by ordered +concatenation of disjoint source ranges with a whole-trajectory digest. READY is +stored once in firstchunk or indexed as a separately bounded common segment; +never duplicate it while claiming disjoint reconstruction. Eight wireindex files +bind ordered chunks, offsets/hash/length and the whole-stream hash. No glob or +runtime-invented declaration. Generic filelimit96 leaves room without exposing +unbounded directories; perfile512MiB ceiling supports the declared320MiB viewport. + +## Wall time and repetitions + +Use measured P5a CPU medians: Jacobi35.357s, SGS25.729s, learned4.699s. +Sum65.785s × fourseeds × fiverepetitions =1,315.700s timed rollout estimate. +Four full Jacobi/learned accuracy pairs add about160.224s before extra capture, +NPZ/JSON/ZIP encoding, warmup, repeated integrity checks and publication. Baseline +is therefore1,475.924s, not an end-to-end measurement. New trained-model timing +may differ; P5a checkpoint performance is evidence for a budget, not a guarantee. + +1800s leaves324.076s (22% over this baseline). That is possible but not a defensible +qualification bound without measuring ~3GiB field/wire/viewport work and runtime +IO. Recommend server-approved **2400s** (40minutes), leaving924.076s (~63%) for +serialization, integrity, scheduling variability and fresh-model changes. This +is a maximum, not an ETA; no wall-time-derived progress. Retain frozenfive repeats. +Use actual observed stages for live status (accuracy completedseed, warmup, +benchmark completedrepetition), reporting original loop progress through narrow +wrapper instrumentation without replacing numerical algorithms. + +Every controller/helper/definition/retained-receipt timeout bound must agree: +validate_definition resource selection, prepare/material validation, captured +helper validate_spec, helper wall deadline and parent stalled-helper deadline, +restart fencing, frontend displayed limits and tests. Readiness10s and ordinary +grace3s/cleanup10s remain separate custody windows. Output verification/publication +also need explicitly bounded parent checks; do not accidentally treat a2400s +process allowance as permission for unbounded post-exit artifact reads. Preserve +small-profile600s behavior, cancellation and unknown-custody semantics. + +If a2400s profile is not admitted, the fallback is separately admitted accuracy +perseed/benchmark perrepetition/aggregate Runs, explicitly changing evaluate +orchestration. Do not automatically split, reduce repetitions, retry or rewrite +the P5b freeze after test observation. + +## Byte accounting and retention + +At32³, allsixNPY element bytes per120-frame trajectory=127,303,680B; +eight trajectories=1,018,429,440B. Accuracy stdout acrossfourJacobi+fourlearned +trajectories≈1,144,258,560B (learned extra RHS); framing adds small measured bytes. +64chunks×256KiB metadata/header reserve gives supplemental chunk estimate +2,179,465,216B. Original eightNPZ uncompressed payload≈766,771,200B; +reserve800MiB compressed NPZ aggregate,320MiB viewport and32MiB reports. +Combined expected-conservative final evaluation output budget D=3,387,424,768 bytes (3.387 GB; approximately3.155 GiB), within4GiB. +Compression exactsize and JSONsize are to be measured, not fabricated acceptance. + +A learned16-frame raw+NPY chunk is38,141,952B before metadata, below40MiB. +Adding another4,194,304B of pressure-input bytes would exceed40MiB; keep that +optional input evidence separate or re-admit shorter chunks. All original stdout +plus allsixfield arrays remain retained; no lossy float reencoding. + +No requirement to import/freeze every evaluation chunk into Data. Preserve all +Run artifacts, import one representative chunk for the real inspection/review/UI +checkpoint, and preserve shared trajectory family protection acrossallchunks. +Training Data records/release remain fully retained from P5b. Shared immutable +captured runtime is stored once. Cleanup only task-owned redundant private +staging after confirmed publication and custody verification; unknown outcomes +retain evidence. For fresh QA backup/restore, preserve verified full backup then +remove only its owned original catalog before restoring elsewhere, allowingtwo +copies instead ofthree. Recompute actual measured peak bytes/free space at final +admission; no proven storage blocker follows from the4GiB cap alone. + +## Required real acceptance + +Completed approved Runs and cancellation/restart evidence; immutable fresh +checkpoint/freeze/training release binding; actual source/runtime/native hashes; +all original NPZ/viewport/result artifacts preserved; independent NumPy reads +compare allsix chunk element bytes to exact retained wire, MAC geometry, frame +indices/times/stages and original NPZ overlappingfields. Independently recompute +velocity/tracer/centered-pressure errors and physics checks from retainedfields; +RHS numerical validation remains distinct from an invented original accuracygate. +Check original report residual≤1e-8/divergence≤1e-5/Courant≤1 and learned0iterations, +finite field/errors and frozen validation gates, full benchmark coverage/rotation, +median/ratio/max/hash arithmetic and originalresult artifact map. Retain evidence +that these are previously exposed seeds and CPU execution; noCUDA/Pumas/historical +training or timing reproduction claim. P5c is incomplete until these real gates +pass; synthetic fixture tests and P5a success do not qualify it. + +## Fresh P5b freeze compatibility, verified against current source + +This is source-semantic inspection while the actual fitting Run is active, not a +claim that a finished freeze/checkpoint exists. Current cpu_train emits a NEW +rheon-trajectory-freeze-v1 with n32/devicecpu,8trainingframes,60validationframes, +120testframes,seeds900–903,fiverepetitions,methodsJacobi/SGS/learned. source contains +actual Torch/NumPy/Python, snapshotSHA, threads, parallelbinary/build hashes and +the four pinned original script hashes. It embeds six trainingrelease identities, +ordered trajectories, fit, validation, positive-diagnostic ablation and memory. + +That source_sha256 map is intentionally not original provenance()'s allRust/Git +inventory. Evaluation's provenance hook must recompute EXACTLY this admitted +four-script map for equality with the immutable fresh freeze; independently bind +native build/Rust/toolchain receipt and captured runtime alongside it. Never +synthesize historical original provenance to make equality pass. exp.BINARY must +point to captured parallelnative before original evaluate's provenance check. +Respect source.threads as recorded by P5b; the original CLI's hardcodedTorch1 is +not called or silently substituted. Verify actual runtime snapshot and Torch +match and CUDA stayshidden before imports. Retain original test count/repetitions. + +P5b result.json binds freezeSHA/checkpointSHA/trainingNPZSHA/inputreceiptSHA, +source/release identities, actualvalidation and native/ablation evidence. Compare +these four direct artifact inputs pairwise before loading checkpoint. Checkpoint +weights_only CPUload must verify originalschema,n32,gains32767,basis32×32, +float64finitepositive gains, CPUplacement. No P5a historicalcheckpoint pin applies +to this newly fitted model. Capture its measured hash from verified P5b authority. + +Exact ownership proposal: root/generic owner recipe_contract inputunion/profile/ +streaming outputverification + Pipeline owner injection; Runs owner bounded +capture_artifact reuse/candidates/streamingpublication/directinput access effects; +helper owner finite2400s profile validation/deadlines; UI owner inputkind Runartifact +selector/roles and shared progress/result render; optionaladapter owner evaluate +hooks/chunkprojection/declarations; independent owner coldNumPy wholewire/chunk/ +originalNPZ/metric verification and true train→evaluate browser/restart evidence. +No edits to runtime sources are authorized by this planning document. + +Additional concrete transport bound: full original evaluation launches68 native +children (8accuracy +60timed), exceeding the current proxy's64 evidence limit. +Admit a closed evaluation mode/cap72 preserving default P5a/P5b64. It authorizes +only32³/seeds900..903/three methods; no open-ended launch allowance. Learned +capture wire requires its additional RHS-before-report framing, unlike the +Jacobi-only producer decoder. Implement an explicitly named learned evaluation +projection in the optionaladapter with independent byte-oracle tests; do not +weaken the old Jacobi decoder or relabel learned output as Jacobi. Enforce fresh +freeze's8GB inference memory budget using actual recorded peak usage and zeroGPU +allocations for this CPUqualification, before declaring verified. + +## Generic Runs, publication and download design review + +Source inspection: /workspace/Tuldok-p5b, during active P5b fitting. This section is a proposal only; no source edits or heavy tests performed. It complements the optional adapter design above and preserves its 512 MiB/file, 4 GiB total, 2400-second proposed bounded profile. This proposal does not assert that such a runtime or disk budget is already qualified. + +### Current costs and authority + +- `recipe_contract.verify_outputs` reads 64 KiB chunks but retains them in a list and joins them, so it is not bounded-memory hashing. `Runs._publish_external` then rereads every entire file into bytes before copying. The numerical recipe's largest array containers would multiply Python RSS unnecessarily. +- `Runs.list()` calls `_verified_artifacts` for every completed Run. `get()` does the same for its selected Run. `_verify_external_artifacts` validates approved output declarations, the durable custody receipt SHA/identity/completed drainage and exact artifact-manifest equality, then rereads every payload entirely. +- `artifact()` calls that full verification and rereads the requested file. `capture_artifact()` likewise verifies everything and rereads the selected payload. `_external_result_summary()` currently verifies all artifacts and calls `artifact()`, repeating full verification again. Downloading N equal-sized artifacts therefore scans O(N²) payload bytes overall; one get itself is O(total output bytes), not O(N²). +- HTTP `Handler.reply` needs a complete bytes value and writes it with Content-Length. `/api/workspace/runs/{id}/artifacts/{name}` calls `Runs.artifact`, so it cannot currently deliver a large output without materializing it. +- Existing nonexternal paths own classification test exposure. External recipes have no classification exposure operation; optional evaluation must keep its previously exposed seed labels. Never make a generic streaming endpoint bypass `TEST_FILES`/`exposure_events` or expose a model/server pickle loader. + +### Smallest shared authority split + +Extract private `Runs._external_inventory(run)` from the existing `_verify_external_artifacts`: validate completed state, `_load` material/spec identity, closed declared inventory and totals, approved roles, required outputs, exact durable manifest, original custody receipt hash and `validate_receipt` completed/drained identity. It returns the existing artifact rows and receipt evidence without certifying payload contents. Verify owned directory chain, regular-file inventory and expected file sizes for cheap availability; this is not a replacement for SHA verification. Reject manifest-only contradictions before touching bulk files. + +Keep `_verify_external_artifacts` as the complete audit operation over `_external_inventory`, using incremental SHA reads. Existing callers requiring all-output acceptance keep it. Add a private selected-artifact verifier that shares `_external_inventory` and hashes only the requested declared regular file. `artifact`, bounded `capture_artifact` and result-summary projection use this selected verifier; checkpoint summary references get their own selected check. No alternate manifest parser, stat-only hash cache, new artifact database or synthesized receipt. Validate every actual selected read freshly; never trust cached file mtimes as content identity. + +Preserve old small-profile list/get all-output verification behavior and refusal semantics. For the new explicitly admitted large-output profile only, list/get run `_external_inventory` and expose `{artifact_status:{status:"receipt_verified",payloads:"not_checked"}}`; if size/inventory checks fail expose existing unavailable diagnostics. This status means completed custody/publication was recorded, not that all bytes were just rehashed. UI lists “Recorded outputs; verified on download” for this case, not “all artifacts verified.” A selected download/capture is fully verified, and an explicit full-audit test/acceptance workflow still hashes all outputs. Do not silently weaken historical small-profile response semantics to optimize the new profile. Full audit remains available internally for restore/acceptance; a new public audit service is unnecessary for P5c. + +For result-summary progress use metadata authority once plus bounded selected summary bytes and selected referenced checkpoint verification; do not rescan giant trajectory files. A valid summary remains recipe-reported and hash-verified, not certification that unrelated full-field outputs remain unaltered. If the full audit later finds damage, show that failure; never repair state or substitute old metrics. + +### Bounded publication + +First make `recipe_contract.verify_outputs` hash incrementally without chunks/join, preserving current return rows/order/sha exactly. Retain flat declared names, regular non-symlink files with link count one, before/after inode/size/mtime/ctime checks, required outputs, file count and actual byte totals. Use at most 1 MiB working buffers. Reject a declared/observed overlimit file before reading it; still count actual read bytes to catch growth. + +Then change `_publish_external` to copy each verified source into an exclusive private destination file in bounded chunks, recomputing size/SHA during the copy and comparing with the first verifier's exact row. Two passes are acceptable O(total bytes) and preserve the current separation of verification and publication; do not introduce a streaming framework merely to combine them. Never link/move mutable producer outputs as published authority. Sources must be under current owned staging with directory-FD/no-follow traversal, not unchecked persisted absolute paths. Fsync completed files, write the same manifest and custody-receipt byte encoding, fsync the publication directory/parent, then persist completed state through existing transaction/custody ownership. Failed/cancelled/partial copies never become completed artifacts. Preserve current failure recovery evidence; remove only newly owned partial publication files under an explicit cleanup outcome, never historical artifacts or source staging needed for custody recovery. + +Existing output manifest and custody receipt schema/bytes remain unchanged. Large profile changes only newly admitted bounds, not retained identity normalization. Coordinate profile limits in recipe definition validation, helper spec validation, RLIMIT_FSIZE and output monitor, retained material/spec decoding, Runs verification and any bounded copy routine. Do not repurpose `recipe_process.OUTPUT_LIMIT` for both larger output limits and captured-input limits: its existing per-capture check is a distinct contract. Keep system/small CPU historical limits unchanged and activate the profile only through reviewed server-approved material, not a client-supplied number alone. + +### Verified streaming downloads + +Add inspect-authorized context-managed `Runs.open_artifact(run_id, name)` returning an owned seekable stream plus existing metadata `{bytes,sha256,media_type}`. Reuse selected artifact/custody verification for external Runs. For nonexternal Runs preserve existing `artifact()`/test exposure behavior first; wrapping its already bounded bytes in a stream is sufficient without changing classification contracts. Reject unknown filenames before filesystem access. There is no arbitrary-path parameter or Range support in this milestone. + +Before sending HTTP success headers, copy the selected external payload into an anonymous temporary file while hashing and checking its declared byte count and source stability; reject mismatch and close/delete spool before sending anything. Rewind the verified spool and stream 64 KiB chunks to `wfile`. This makes downloaded bytes exactly the verified bytes even if the original is mutated after verification; prehashing then streaming the same writable source FD would leave that race. Memory remains bounded, with at most one additional selected-file disk copy per download (up to 512 MiB; this recipe’s largest declaration is the 320 MiB viewport). Close spool/FDs on success, authorization failure, exceptions and client disconnect. Ordinary data is not buffered into `reply()`. + +App owns a small `reply_stream` helper: same Content-Type, Content-Length, Cache-Control:no-store and X-Content-Type-Options:nosniff, no user-controlled headers; context lifetime covers transmission. Resolve auth and all integrity checks before 200 headers. Post-header I/O/disconnect aborts transport and never writes a JSON error into binary output. Avoid holding the catalog lock/transaction while copying or serving bytes. Bound concurrent large downloads with a small process-local permit (one admitted large download at a time), reject capacity before work with typed unavailable response, and release in finally; this is I/O admission, not a scheduler. Keep authorization valid at method entry and never create a bearer download URL. + +Disk qualification includes staging outputs + immutable published outputs (up to two full 4 GiB sets during publication) + one at-most-512 MiB download spool, existing CPU snapshot and training inputs. Check measured free space before admission/publication and spool creation; retain authoritative artifacts on ENOSPC. The cap is not proof the environment has sufficient capacity. + +### Completed-Run input owner and candidates + +Preserve existing P4 `capture_artifact(run_id,name,expected_sha256)` callers and exact returned source receipt keys. Add optional keyword-only `max_bytes` defaulting to the existing 64 MiB external artifact bound for legacy callers; validate selected inventory size against it BEFORE reading any selected payload. The generic input owner passes its declared per-slot bound, which remains at most 40 MiB in P5c; four evaluation inputs are each at most 4 MiB, so no streaming capture union is needed. Refactor this method to the selected verifier rather than all-output rescans, and preserve its raw custody receipt base64/hash and originating run/attempt/material identity exactly. + +Extend `recipe_contract.prepare(..., *, releases=None, runs=None)` and all Pipeline call sites with Runs owner injection. New binding is exactly `{kind:"run_artifact",run_id,artifact_name,expected_sha256}`; new material receipt variant is `{kind:"run_artifact",source:}`. Validate same catalog, bytes/hash/capture equality, closed source receipt and completed custody evidence during retained reads. Old project/frozen_release receipt bytes stay unchanged. Requests cannot supply provenance or a filesystem path. The optional adapter verifies semantic roles and all four inputs come from the same completed fit with mutually agreeing hashes; generic core owns custody rather than Rheon meaning. + +Add inspect-authorized `Runs.input_candidates()` and `GET /api/workspace/run-artifact-inputs`, returning `{runs:[{run_id,state:"completed",artifacts:[{artifact_name,role,bytes,sha256,status}]}]}` for at most 100 completed external Runs and at most the profile-declared outputs each (32 legacy, 96 large profile). Candidates inspect `_external_inventory` without reading giant payloads; mark `status:"receipt_verified"`, never content-verified. Exclude over-40 MiB input candidates, preserve empty eligible lists, and return a bounded unavailable entry/reason for corrupted run inventory rather than hiding why it is unusable. Use closed Run entries `{run_id,state,status,reason,artifacts}`: status `receipt_verified`/`unavailable`, reason null/bounded string, artifacts empty on unavailable. Artifact entries have the five fields above and status `receipt_verified`; unavailable Runs do not publish untrusted artifact names. Editing selected Run/artifact/hash invalidates saved preflight and approval; capture rechecks actual bytes before GO. No candidate endpoint opens an artifact or exposes classification holdout data. + +The existing Access external-submission narrow inspection whitelist must be reviewed: add only `runs.capture_artifact` for precisely permitted preflight/submission scope if required by its existing mode, not global execute/read grants or job-mode broadening. Ordinary candidates require inspect. Failed/cancelled/unknown sources cannot become inputs. Lost acknowledgement continues through Runs' existing request-key/actor/digest reconciliation. + +### Ownership and decisive tests + +Root: recipe_contract streaming verification/profile/binding/receipt/token and Pipeline injection; Runs owner: inventory/selected/full validators, publication/download/capture/candidates, app stream route, focused new `tests/test_external_artifacts.py`; helper owner: consistent optional profile/time/output monitor bounds; UI owner: explicit inventory-vs-payload verification labels and Run-artifact selector; numerical adapter owner: unchanged algorithms and independent full-field qualification above. + +Light tests before real evaluation: old receipts/manifest bytes unchanged; instrumented bounded reads/no read_bytes/no chunk accumulation; N requested artifacts hash O(total selected bytes) rather than N×all bytes; list/get old behavior retained and new-profile explicit not_checked status; all required metadata/receipt mismatches rejected; selected/nonselected tamper meanings; same-size/in-place mutation during copy; symlink/FIFO/hardlink/directory replacement; oversize/growth/shortreads; ENOSPC and fsync/publish rollback; cancelled/unknown outputs never downloadable/capturable; real HTTP checksum/Content-Length/auth/disconnect cleanup; test exposure refuses before any stream; relocated restore uses current owned paths and exact historical receipt; candidate metadata never substitutes for capture; role/hash/same-originating-fit disagreements rejected. Use bounded synthetic payloads for resource logic, not multi-GiB fixtures. The real evaluation acceptance independently streams/hashes every retained artifact and compares full fields; no sampled checker substitutes for that claim. + + +### Corrected disk budget and selected publication strategy + +Selected strategy: bounded streaming-copy publication, preserving current lifecycle. Approved disposable Rust-installer cleanup removed 1.685 GB; parent reported 8.362 GB free afterward. No completed-run duplicate cleanup has been performed or assumed. Corrected D is 3,387,424,768 bytes (3.387 decimal GB, approximately 3.155 GiB), not 3.39 GiB. Two copies are 6,774,849,536 bytes; against 8,362,000,000 free this leaves 1,587,150,464 bytes for temporary activity/headroom. D is the current conservative output estimate, not measured completed-evaluation bytes. + +The largest recipe-declared artifact is viewport.json, at most 320 MiB=335,544,320 bytes. With one concurrent download, actual recipe spool peak is therefore at most 335,544,320 bytes; no full-output spool is made. Profile-generic maximum is 512 MiB=536,870,912 bytes for another approved large recipe. At the parent-reported free-space point, two D copies plus the recipe spool leave 1,251,606,144 bytes; using the generic512MiB spool leaves 1,050,279,552 bytes. The existing CPU snapshot, training catalog and other existing data already reduce reported free space; do not count them a second time. Future inputs, reports and new artifacts consume the remaining budget and must be measured. + +Read-only df in this review observed /workspace free8,208,728,064 bytes, reflecting ongoing workspace activity. At that observation, two D copies leave1,433,878,528 bytes; generic512MiB spool leaves897,007,616 bytes. Thus report times matter and final admission must remeasure. Proposed reserve: one maximum512MiB spool plus512MiB unallocated headroom; 2D+1GiB=7,848,591,360 bytes, below that observed free space by360,136,704 bytes. This is provisional capacity, not a reservation guarantee against unrelated same-filesystem writers. The /tmp filesystem reports separately; place anonymous spool explicitly on the measured catalog filesystem so budget assumptions do not silently depend on /tmp behavior. + +At admission, compare actual free space to this recipe's bounded projected output+publication growth and reserve; before copying compare remaining publication bytes to free space again; before download reserve the selected size within the single-spool permit. Do not require 2×the universal4GiB cap for every smaller admitted recipe: use its exact finite admitted budget and observed staging bytes. Concrete proposal for this evaluation recipe: `resources.max_output_bytes=3_500_000_000` (below the generic4GiB ceiling); two such maximum copies plus1GiB reserve total8,073,741,824 bytes, fitting the observed8,208,728,064 by134,986,240 bytes. This margin requires a fresh admission check and does not guarantee output compression will fit; limit failure must remain honest rather than discarding fields. Conversely D is only an estimate until a finite declared aggregate bound justifies it; the recipe must declare a fitting aggregate limit or demonstrate the4GiB worst-case fits before running. ENOSPC remains typed failure preserving original staging/receipt and unknown-commit evidence. No streaming buffers or cleanup assumptions can substitute for actual filesystem capacity. + +Atomic rename remains an UNADMITTED alternative only if measured storage prevents the selected copy path. It would require a separately reviewed ownership-transfer/recovery decision: confirmed drainage, exact verified output directory, same filesystem, exclusive destination, unchanged historical spec/receipt identities, fsync ordering and crash-after-rename-before-DB reconciliation without replay. No rename implementation is in this milestone's selected write set. Copy-plus-delete-per-file is likewise excluded because it partially destroys the source set during publication. No deletion of existing completed duplicates is authorized by this capacity calculation. + +For later MCP chunk reads, reuse this selected-artifact owner; do not call full-artifact verification for every chunk. Do not claim each independent chunk proves whole-file identity unless it comes from a bounded verified spool/snapshot with explicit handle lifetime or a separately specified authenticated chunk manifest. P5c needs ordinary complete downloads only; defer a chunk protocol rather than introduce one implicitly. + + +### Final proposed file ownership for selected copy-only implementation + +- Generic recipe/Pipeline owner: `recipe_contract.py`, `pipeline.py`, `tests/test_recipe_contract.py`, `tests/test_cpu_recipe_contract.py`; profile-specific limits, chunked output hashing, exact run_artifact union, immutable receipt validation and generated input evidence. `access.py` plus `tests/test_access.py` only if the existing narrow submission capability actually requires the new capture action; do not add speculative permissions. +- Runs/HTTP owner: `runs.py`, `app.py`, new `tests/test_external_artifacts.py`, and affected `tests/test_external_progress.py`/`tests/test_external_http.py`; shared inventory/selected/full verification, bounded streaming copy publication, single-permit verified anonymous-spool downloads, bounded prior-Run capture/candidates, summary avoiding unrelated large payload rehash. Keep `_publish_external` lifecycle/manifest encoding and existing small-profile/test exposure behavior. No new generic storage framework/file needed. +- Helper owner: `recipe_process.py`, `tests/test_recipe_process.py`; declared large profile2400s/96outputs/512MiBfile/4GiBaggregate caps as admitted, with existing small-profile limits preserved and capture-input limits independently enforced. +- Optional numerical adapter owner: new `adapters/rheon/evaluate.py`, `adapters/rheon/evaluation_project.py`, new `tests/test_rheon_evaluation.py`; existing native transport helper only for the already declared capture/provenance hooks. Upstream numerical source remains unchanged. +- UI/evidence owner: `static/workspace-external.js`, `static/workspace-real.js`, `tests/test_workspace_external_controller.cjs`, `tests/browser_workspace_external.cjs`, new `tests/browser_workspace_rheon_evaluation.cjs`; Run-artifact picker and verification-scope labels, real approved train→evaluate follow-through. Reuse existing polling/auth/helpers rather than a second telemetry path. +- Root durable documentation: `docs/contracts/external-recipes.md`, `docs/contracts/captured-cpu-runtime.md`, new `docs/contracts/rheon-evaluation.md`, and parent plan/ledger/issues/admission/verification reports. No schema migration, database table, MCP chunk API or artifact receipt schema rewrite is needed for this slice. + +This design update changed only /tmp/tuldok-p5c-admission.md. No publication code, deletion, evaluation or heavy test was performed. + +### Shared cold source receipt validation + +The existing scientific Run-source importer already validates retained custody +receipts. Move that unchanged pure receipt check into recipe_process, the existing +cold custody owner, and delegate from scientific_fields; recipe_contract calls +the same function for new Run-artifact inputs. Preserve source schema, receipt +bytes and existing scientific error semantics. This narrow scientific_fields.py +write avoids duplicate validators and introduces no new parser or service. +Inspection of Access confirms ordinary inspect scope already covers capture; no +new permission, whitelist or owner bypass is needed. + +## Source preparation and capacity checkpoint + +Independent core review found a legacy small-profile read regression; the owner +restored full verification on all historical read surfaces and added an +unrelated-artifact tamper regression. No remaining core or numerical source +blocker was reported. Focused helper tests passed (27), artifact/progress checks +passed (13), generic checks passed after correcting an authored project-ID fixture, +and all 11 cold-release checks passed with the qualified reader. Nine optional +evaluator tests and source-only assembly qualified declarations, not model work. +The CI training interpreter now also covers evaluation adapter tests. + +The freshly assembled evaluation project captures nine exact current sources, +89 required outputs and the existing runtime snapshot. Failed/interrupted +attempts retain private original outputs and wires under their owned attempt +directory; only successful verified-copy retention removes private duplicates. +This resolves the earlier proposal's separate /tmp temporary-storage assumption. + +After exact-inventory approval, 111 task-owned duplicate files and 16 empty +staging output directories were removed. Every authoritative catalog payload +remains retained; all 20 Run states and all 16 completed publication/custody/hash +sets were reverified. Failed/cancelled staging fingerprints were unchanged. +[Cleanup receipt](p5c/duplicate-cleanup-receipt.json) binds the approved inventory. +Measured free space afterward was 8,301,801,472 bytes, 228,059,648 bytes above the +8,073,741,824-byte admission guard. This is a checkpoint, not a disk reservation; +recheck immediately before admission. Actual evaluation remains gated on P5b +final regression/hosted checks and that fresh capacity observation. diff --git a/docs/plans/generic-scientific-projects/reports/p5c/assembly.json b/docs/plans/generic-scientific-projects/reports/p5c/assembly.json new file mode 100644 index 00000000..80200760 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5c/assembly.json @@ -0,0 +1,14 @@ +{ + "qualification": "Source-only immutable evaluation assembly; no native/model execution", + "project_id": "a32555aabdee479ab94a9cb4159f5a73", + "recipe_sha256": "777fccef814adfeb7afcf599cb56e31faf361b4c6afbd3e80b5b899c87ce271b", + "sources": 9, + "outputs": 89, + "scientific_chunks": 64, + "resources": { + "max_output_bytes": 3500000000, + "profile": "captured_cpu_large_outputs_v1", + "wall_seconds": 2400 + }, + "runtime_snapshot_sha256": "cc54ef199fc192b70f702e85dda70bfc8697bf3220373ae9f031afd8b237f0a7" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-inventory.json b/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-inventory.json new file mode 100644 index 00000000..cb267767 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-inventory.json @@ -0,0 +1,2282 @@ +{ + "schema_version": 1, + "created_utc": "2026-10-13T20:19:17.306604+00:00", + "dry_run": true, + "catalog": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog", + "free_bytes": 7465734144, + "items": [ + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333301, + "mtime_ns": 1791921360613699044, + "ctime_ns": 1791921360613699044, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/fields.zip", + "bytes": 17023873, + "allocated_bytes": 17027072, + "sha256": "dcd360f9f6c68c557e610385745f1f036447848aadb27126a68b729d273d667e", + "device": 27, + "inode": 1333298, + "mtime_ns": 1791921360599048246, + "ctime_ns": 1791921360599048246, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/fields.zip", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/producer-receipt.json", + "bytes": 4242, + "allocated_bytes": 8192, + "sha256": "7243d456b751863acc9b0e69525ffa255d473d609d7c0b2f1814805fa2f28849", + "device": 27, + "inode": 1333300, + "mtime_ns": 1791921360613699044, + "ctime_ns": 1791921360613699044, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/producer-receipt.json", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/wire.bin", + "bytes": 8489920, + "allocated_bytes": 8491008, + "sha256": "886d70ea42c70216ab285fde707b765c2fa81eeb1a10b4f4d5f10c371bb71bb4", + "device": 27, + "inode": 1333299, + "mtime_ns": 1791921360607545788, + "ctime_ns": 1791921360607545788, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/wire.bin", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330066, + "mtime_ns": 1791921288640869511, + "ctime_ns": 1791921288640869511, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/native-diagnostics.txt", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/fields.zip", + "bytes": 17023877, + "allocated_bytes": 17027072, + "sha256": "b591b00b9b9c32f3fe595d03197881f7626adeb89db2cacb2712900f52efe0c0", + "device": 27, + "inode": 1330018, + "mtime_ns": 1791921288626357441, + "ctime_ns": 1791921288626357441, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/fields.zip", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/producer-receipt.json", + "bytes": 4244, + "allocated_bytes": 8192, + "sha256": "f52d752fc048b5708208e6b75565ff01b0a0d9943c84f7a2391497090892bc24", + "device": 27, + "inode": 1330050, + "mtime_ns": 1791921288640869511, + "ctime_ns": 1791921288640869511, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/producer-receipt.json", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/wire.bin", + "bytes": 8489922, + "allocated_bytes": 8491008, + "sha256": "c583e7b9a00b057c1b2dea6f23dbf41c3770fc690a5e7430d8fe8558c733b481", + "device": 27, + "inode": 1330034, + "mtime_ns": 1791921288635398681, + "ctime_ns": 1791921288635398681, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/wire.bin", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328231, + "mtime_ns": 1791921328692320338, + "ctime_ns": 1791921328692320338, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/native-diagnostics.txt", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/fields.zip", + "bytes": 17023839, + "allocated_bytes": 17027072, + "sha256": "c9ceee9e50861ee589b3e1c324d5dcb187cf4f79322fd12d208e450e86544cc1", + "device": 27, + "inode": 1328295, + "mtime_ns": 1791921328536096615, + "ctime_ns": 1791921328536096615, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/fields.zip", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/producer-receipt.json", + "bytes": 4225, + "allocated_bytes": 8192, + "sha256": "8fde547aed7066c30f708e555e1d94c6685d68c8972d8e4926a2201d0f4344df", + "device": 27, + "inode": 1328230, + "mtime_ns": 1791921328692320338, + "ctime_ns": 1791921328692320338, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/producer-receipt.json", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/wire.bin", + "bytes": 8489903, + "allocated_bytes": 8491008, + "sha256": "fff88c99b9d04ebf30440e66155fe4e750083295989bca635ab730e6498f64f2", + "device": 27, + "inode": 1328229, + "mtime_ns": 1791921328609187692, + "ctime_ns": 1791921328609187692, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/wire.bin", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330578, + "mtime_ns": 1791921294915387085, + "ctime_ns": 1791921294915387085, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/native-diagnostics.txt", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "334df086cd8cd6dc9689cf5faafd3631a2b0967b0b216c8a507c57083c2f5dee", + "device": 27, + "inode": 1330530, + "mtime_ns": 1791921294896227524, + "ctime_ns": 1791921294896227524, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/fields.zip", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/producer-receipt.json", + "bytes": 4233, + "allocated_bytes": 8192, + "sha256": "eac0ba8fe582c5f83bd0973acca7be44068e2e799c74340f34764bd421bf2924", + "device": 27, + "inode": 1330562, + "mtime_ns": 1791921294915387085, + "ctime_ns": 1791921294915387085, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/producer-receipt.json", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "5dfafc4343e3d6e1b112bc0ca572d30ff129c0888288e5fc1a04836c824c127d", + "device": 27, + "inode": 1330546, + "mtime_ns": 1791921294907021730, + "ctime_ns": 1791921294907021730, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/wire.bin", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/result.json", + "bytes": 5067, + "allocated_bytes": 8192, + "sha256": "62ccabd6ca01966992665573fbac85c608c9dfbd358ff4e1e6715e50688e4394", + "device": 27, + "inode": 1211665, + "mtime_ns": 1791919167930709239, + "ctime_ns": 1791919167930709239, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/result.json", + "run_id": "44272474e32544898feb39f651d997b9" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/checkpoint.pt", + "bytes": 2874, + "allocated_bytes": 4096, + "sha256": "7d44cfb241c67e0d2b5349c5dbf5b0251a4ea6f4bc3f5f25177c4ecfc6e6911b", + "device": 27, + "inode": 1211649, + "mtime_ns": 1791919167907955432, + "ctime_ns": 1791919167907955432, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/checkpoint.pt", + "run_id": "44272474e32544898feb39f651d997b9" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/array.npy", + "bytes": 224, + "allocated_bytes": 4096, + "sha256": "d4527f6b3061eb636796c8343fa55690843b423063c32c4506be611a678d9fc2", + "device": 27, + "inode": 1211633, + "mtime_ns": 1791919167907955432, + "ctime_ns": 1791919167907955432, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/array.npy", + "run_id": "44272474e32544898feb39f651d997b9" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331090, + "mtime_ns": 1791921301134140883, + "ctime_ns": 1791921301134140883, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/native-diagnostics.txt", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/fields.zip", + "bytes": 17023761, + "allocated_bytes": 17027072, + "sha256": "ea5653a7f628cf292b2bb8c86737e900d4dc711eb197eca8683a5a8344ea03b2", + "device": 27, + "inode": 1331042, + "mtime_ns": 1791921301119205295, + "ctime_ns": 1791921301119205295, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/fields.zip", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/producer-receipt.json", + "bytes": 4186, + "allocated_bytes": 8192, + "sha256": "67352c1ca8559250bbc57b408bc8f32517fc0dd173307ac53f0a155e1e73b187", + "device": 27, + "inode": 1331074, + "mtime_ns": 1791921301133895156, + "ctime_ns": 1791921301133895156, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/producer-receipt.json", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/wire.bin", + "bytes": 8489864, + "allocated_bytes": 8491008, + "sha256": "b71ecbad87f3ccdf3ddc10079db639a91e5990075c32e73a427ab741968a61b6", + "device": 27, + "inode": 1331058, + "mtime_ns": 1791921301128743451, + "ctime_ns": 1791921301128743451, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/wire.bin", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328440, + "mtime_ns": 1791921336824630654, + "ctime_ns": 1791921336824630654, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/native-diagnostics.txt", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/fields.zip", + "bytes": 17023819, + "allocated_bytes": 17027072, + "sha256": "3043098224dc4fedf7ba737e0238b785fa136c02816ee1297a00ccd6524680bc", + "device": 27, + "inode": 1328535, + "mtime_ns": 1791921336697705402, + "ctime_ns": 1791921336697705402, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/fields.zip", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/producer-receipt.json", + "bytes": 4215, + "allocated_bytes": 8192, + "sha256": "8f1f0b8d51cdbc2e1e14609fd75fca5c47451f22e6afddac0c9756456bf5c97c", + "device": 27, + "inode": 1328439, + "mtime_ns": 1791921336824630654, + "ctime_ns": 1791921336824630654, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/producer-receipt.json", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/wire.bin", + "bytes": 8489893, + "allocated_bytes": 8491008, + "sha256": "db690f103de97e19d2de68beebe6d735415a8216a21f1c35559634944cd0802a", + "device": 27, + "inode": 1328438, + "mtime_ns": 1791921336761630266, + "ctime_ns": 1791921336761630266, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/wire.bin", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/result.json", + "bytes": 1007942, + "allocated_bytes": 1011712, + "sha256": "d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448", + "device": 27, + "inode": 1211462, + "mtime_ns": 1791919933887390351, + "ctime_ns": 1791919933887390351, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/result.json", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/timing.json", + "bytes": 871725, + "allocated_bytes": 872448, + "sha256": "0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b", + "device": 27, + "inode": 1211466, + "mtime_ns": 1791919933891429998, + "ctime_ns": 1791919933891429998, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/timing.json", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/input-model.pt", + "bytes": 272149, + "allocated_bytes": 274432, + "sha256": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "device": 27, + "inode": 1211463, + "mtime_ns": 1791919933889157378, + "ctime_ns": 1791919933889157378, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/input-model.pt", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/comparison-receipt.json", + "bytes": 26712, + "allocated_bytes": 28672, + "sha256": "d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a", + "device": 27, + "inode": 1211467, + "mtime_ns": 1791919933892556664, + "ctime_ns": 1791919933892556664, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/comparison-receipt.json", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/input-freeze.json", + "bytes": 96833, + "allocated_bytes": 98304, + "sha256": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "device": 27, + "inode": 1211464, + "mtime_ns": 1791919933889928431, + "ctime_ns": 1791919933889928431, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/input-freeze.json", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/comparison-freeze.json", + "bytes": 1400, + "allocated_bytes": 4096, + "sha256": "5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076", + "device": 27, + "inode": 1211465, + "mtime_ns": 1791919933890396177, + "ctime_ns": 1791919933890396177, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/comparison-freeze.json", + "run_id": "0cc3ef6b5f9e460f873c8eda1ba3d525" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331602, + "mtime_ns": 1791921308198352572, + "ctime_ns": 1791921308198352572, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/native-diagnostics.txt", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/fields.zip", + "bytes": 17023801, + "allocated_bytes": 17027072, + "sha256": "64620a51df29a223938c6db43a2450296e412e60dff944ba128df5f8e3df8605", + "device": 27, + "inode": 1331554, + "mtime_ns": 1791921308181198216, + "ctime_ns": 1791921308181198216, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/fields.zip", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/producer-receipt.json", + "bytes": 4205, + "allocated_bytes": 8192, + "sha256": "f6abe10655bfd3ac050b5755c5c745d6b403983592d8f5905952dead31b82244", + "device": 27, + "inode": 1331586, + "mtime_ns": 1791921308198352572, + "ctime_ns": 1791921308198352572, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/producer-receipt.json", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/wire.bin", + "bytes": 8489885, + "allocated_bytes": 8491008, + "sha256": "737208512c9f32591d0bd125b2c5eab871827f395af6e5ce10eeb6a03c9a82dd", + "device": 27, + "inode": 1331570, + "mtime_ns": 1791921308190069441, + "ctime_ns": 1791921308190069441, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/wire.bin", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328374, + "mtime_ns": 1791921258704198126, + "ctime_ns": 1791921258704198126, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/native-diagnostics.txt", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/fields.zip", + "bytes": 17023815, + "allocated_bytes": 17027072, + "sha256": "d801973641c7ebd42b193c37b9c06053d0db59a4ab34dcfd7fcddf8a82d53600", + "device": 27, + "inode": 1328371, + "mtime_ns": 1791921258693723739, + "ctime_ns": 1791921258693723739, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/fields.zip", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/producer-receipt.json", + "bytes": 4213, + "allocated_bytes": 8192, + "sha256": "9019802f4a742730e9b63a09718480af30cd3ecbb87924938886d10439ecfca5", + "device": 27, + "inode": 1328373, + "mtime_ns": 1791921258704198126, + "ctime_ns": 1791921258704198126, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/producer-receipt.json", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/wire.bin", + "bytes": 8489891, + "allocated_bytes": 8491008, + "sha256": "851969c2e6cfe8782078247271a13b950dcc26e48201a6e3af76bb03b844e4b6", + "device": 27, + "inode": 1328372, + "mtime_ns": 1791921258700259032, + "ctime_ns": 1791921258700259032, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/wire.bin", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328679, + "mtime_ns": 1791921345725735069, + "ctime_ns": 1791921345725735069, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/native-diagnostics.txt", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/fields.zip", + "bytes": 17023745, + "allocated_bytes": 17027072, + "sha256": "93e4dd14ea507948ab329a08ae86282d4548422e59fa88d2644e439f1138db1d", + "device": 27, + "inode": 1332626, + "mtime_ns": 1791921345704804918, + "ctime_ns": 1791921345704804918, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/fields.zip", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/producer-receipt.json", + "bytes": 4178, + "allocated_bytes": 8192, + "sha256": "377b8a1c150f9c11ab1a41f0b7c9d80cc6913c5c861c0951e17d83a7cedf0164", + "device": 27, + "inode": 1328678, + "mtime_ns": 1791921345725735069, + "ctime_ns": 1791921345725735069, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/producer-receipt.json", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/wire.bin", + "bytes": 8489856, + "allocated_bytes": 8491008, + "sha256": "8fe7fd42d00fd96007941a853d1cacd6324b626e3b090cab4d86c7078ef8a713", + "device": 27, + "inode": 1332642, + "mtime_ns": 1791921345715187715, + "ctime_ns": 1791921345715187715, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/wire.bin", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328551, + "mtime_ns": 1791921282535539099, + "ctime_ns": 1791921282535539099, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/native-diagnostics.txt", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/fields.zip", + "bytes": 17023863, + "allocated_bytes": 17027072, + "sha256": "1c137295d187099956c0b96149e0cd1babaab893d2d7396e563593d8abc85028", + "device": 27, + "inode": 1328611, + "mtime_ns": 1791921282524654254, + "ctime_ns": 1791921282524654254, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/fields.zip", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/producer-receipt.json", + "bytes": 4237, + "allocated_bytes": 8192, + "sha256": "af4d5a0e2b42bdd62b05db96bdc4cd8b9d85bbdd0028bb8b23a3dac387a5080d", + "device": 27, + "inode": 1328550, + "mtime_ns": 1791921282535539099, + "ctime_ns": 1791921282535539099, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/producer-receipt.json", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/wire.bin", + "bytes": 8489915, + "allocated_bytes": 8491008, + "sha256": "3d007c36d1d79743f39940aec60b8908314dc8baae9e71fc7e93fe6148cc16bd", + "device": 27, + "inode": 1328549, + "mtime_ns": 1791921282532027576, + "ctime_ns": 1791921282532027576, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/wire.bin", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1332098, + "mtime_ns": 1791921314654730691, + "ctime_ns": 1791921314654730691, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/native-diagnostics.txt", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/fields.zip", + "bytes": 17023919, + "allocated_bytes": 17027072, + "sha256": "c0b9c23b226ca261393aa741358ea3c03e476ce80bfae1dc956852730cee2948", + "device": 27, + "inode": 1332050, + "mtime_ns": 1791921314642920294, + "ctime_ns": 1791921314642920294, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/fields.zip", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/producer-receipt.json", + "bytes": 4265, + "allocated_bytes": 8192, + "sha256": "4152d0d691ccc78099b048081059a2b2608143e317ea6a0b322c2261fe366130", + "device": 27, + "inode": 1332082, + "mtime_ns": 1791921314654730691, + "ctime_ns": 1791921314654730691, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/producer-receipt.json", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/wire.bin", + "bytes": 8489943, + "allocated_bytes": 8491008, + "sha256": "ca94a63ef1a73229218199860a5391e858e766874ebc1b7362a06f65228d4aad", + "device": 27, + "inode": 1332066, + "mtime_ns": 1791921314649425610, + "ctime_ns": 1791921314649425610, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/wire.bin", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/result.json", + "bytes": 129795, + "allocated_bytes": 131072, + "sha256": "9b680ad1780fe276b5385805a06dcc3acb365f6b3475481af20e982e6495aa79", + "device": 27, + "inode": 1328361, + "mtime_ns": 1791921788687566151, + "ctime_ns": 1791921788687566151, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/result.json", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/model.pt", + "bytes": 272149, + "allocated_bytes": 274432, + "sha256": "e08e85d991d2734383d14501d6dcfdaccb45dfd53370d883c26b44e84255f5f8", + "device": 27, + "inode": 1328351, + "mtime_ns": 1791921747643555860, + "ctime_ns": 1791921747643555860, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/model.pt", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/training.npz", + "bytes": 50332156, + "allocated_bytes": 50335744, + "sha256": "3ca87734e4ebe41320328b130908dc88b85967dcc745f80f8c207cf29b539184", + "device": 27, + "inode": 1328350, + "mtime_ns": 1791921747423559147, + "ctime_ns": 1791921747423559147, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/training.npz", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/freeze.json", + "bytes": 79042, + "allocated_bytes": 81920, + "sha256": "020ac431def0540edca65b6009ee77ebf8a5eff414efd9cf7da38532ef711792", + "device": 27, + "inode": 1328359, + "mtime_ns": 1791921788684229056, + "ctime_ns": 1791921788684229056, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/freeze.json", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/result-summary.json", + "bytes": 611, + "allocated_bytes": 4096, + "sha256": "ad89e6106e83a06a6ac68dd0eb950338cf05ffba768f3fd72558cb03348188ec", + "device": 27, + "inode": 1328362, + "mtime_ns": 1791921788687926230, + "ctime_ns": 1791921788687926230, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/result-summary.json", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/input-receipt.json", + "bytes": 55329, + "allocated_bytes": 57344, + "sha256": "1a8ec140ce18fee118ba9821353a2538b3caf8ab89b73a678bd8acdf03bed189", + "device": 27, + "inode": 1328360, + "mtime_ns": 1791921788684938458, + "ctime_ns": 1791921788684938458, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/input-receipt.json", + "run_id": "11fc2be1aada490f9b67c11c190728f6" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1327880, + "mtime_ns": 1791921321460444827, + "ctime_ns": 1791921321460444827, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/native-diagnostics.txt", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "7ad4b8d7e5de6b4a3aec1539062661df7551f74f66f0fe92285802dad3e99cd9", + "device": 27, + "inode": 1327877, + "mtime_ns": 1791921321438815560, + "ctime_ns": 1791921321438815560, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/fields.zip", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/producer-receipt.json", + "bytes": 4233, + "allocated_bytes": 8192, + "sha256": "412a4936341eb2e6f0555135707fbc5fe37bba218c30d8132b30c80f84cf904a", + "device": 27, + "inode": 1327879, + "mtime_ns": 1791921321460444827, + "ctime_ns": 1791921321460444827, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/producer-receipt.json", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "6cc696471fe628586099d19f15b2202093ffebdcd08e6b0c27663473b0cc8a51", + "device": 27, + "inode": 1327878, + "mtime_ns": 1791921321453223711, + "ctime_ns": 1791921321453223711, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/wire.bin", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333058, + "mtime_ns": 1791921353144008800, + "ctime_ns": 1791921353144008800, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/native-diagnostics.txt", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/fields.zip", + "bytes": 17023869, + "allocated_bytes": 17027072, + "sha256": "54f44df2fe839f283511d125a5000138070243552504df883b349455d63e46c3", + "device": 27, + "inode": 1333010, + "mtime_ns": 1791921353079162178, + "ctime_ns": 1791921353079162178, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/fields.zip", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/producer-receipt.json", + "bytes": 4239, + "allocated_bytes": 8192, + "sha256": "18a5c3e99bff019f6839e721180e5fa25c13ce97369f3993721dc3910aef597a", + "device": 27, + "inode": 1333042, + "mtime_ns": 1791921353144008800, + "ctime_ns": 1791921353144008800, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/producer-receipt.json", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/wire.bin", + "bytes": 8489919, + "allocated_bytes": 8491008, + "sha256": "d0fc71906b63ed46038380f1ac09f211e48b4a44d4ce86dfd47210f9eaa2c259", + "device": 27, + "inode": 1333026, + "mtime_ns": 1791921353138165648, + "ctime_ns": 1791921353138165648, + "category": "completed_run_output", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/wire.bin", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-0.zip", + "bytes": 34145326, + "allocated_bytes": 34148352, + "sha256": "c21553baf38d2aa44a17eba441d039d9c6c22e863ec436c1e7e524ce923d0f14", + "device": 27, + "inode": 1333411, + "mtime_ns": 1791921365925211368, + "ctime_ns": 1791921365925211368, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/c21553baf38d2aa44a17eba441d039d9c6c22e863ec436c1e7e524ce923d0f14.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-1.zip", + "bytes": 34145202, + "allocated_bytes": 34148352, + "sha256": "e3dbff795de37df43501fc26010635242f600d3c3e228188bd2e63dbf17066b6", + "device": 27, + "inode": 1333442, + "mtime_ns": 1791921368785169789, + "ctime_ns": 1791921368785169789, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/e3dbff795de37df43501fc26010635242f600d3c3e228188bd2e63dbf17066b6.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-2.zip", + "bytes": 34145306, + "allocated_bytes": 34148352, + "sha256": "4e7bdc2b16554a6c73321bbd15329408a39f710a02ef34fc8b29507493bd8e0e", + "device": 27, + "inode": 1333458, + "mtime_ns": 1791921371889124511, + "ctime_ns": 1791921371889124511, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/4e7bdc2b16554a6c73321bbd15329408a39f710a02ef34fc8b29507493bd8e0e.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-3.zip", + "bytes": 34145280, + "allocated_bytes": 34148352, + "sha256": "fd1e3412b8ce639ece6821f69f3981127d94052a88ab28649844d630b5038c4c", + "device": 27, + "inode": 1333474, + "mtime_ns": 1791921374713083905, + "ctime_ns": 1791921374713083905, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/fd1e3412b8ce639ece6821f69f3981127d94052a88ab28649844d630b5038c4c.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-4.zip", + "bytes": 34145150, + "allocated_bytes": 34148352, + "sha256": "296f07e01ab0cf0dc428f9b680b0d02026705494847b9ee23845d46ce66ec893", + "device": 27, + "inode": 1327883, + "mtime_ns": 1791921377285046219, + "ctime_ns": 1791921377285046219, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/296f07e01ab0cf0dc428f9b680b0d02026705494847b9ee23845d46ce66ec893.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-5.zip", + "bytes": 34145328, + "allocated_bytes": 34148352, + "sha256": "06e843c74f9cdda576a52859b736435eb6c8aee7f87b565de8dcc6625824c64b", + "device": 27, + "inode": 1327885, + "mtime_ns": 1791921380205003565, + "ctime_ns": 1791921380205003565, + "category": "qa_release_zip", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/releases/06e843c74f9cdda576a52859b736435eb6c8aee7f87b565de8dcc6625824c64b.zip", + "run_id": null + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/fields.zip", + "bytes": 17023863, + "allocated_bytes": 17027072, + "sha256": "1c137295d187099956c0b96149e0cd1babaab893d2d7396e563593d8abc85028", + "device": 27, + "inode": 1328643, + "mtime_ns": 1791921283302396815, + "ctime_ns": 1791921283302396815, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/fields.zip", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328691, + "mtime_ns": 1791921283598392574, + "ctime_ns": 1791921283598392574, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/wire.bin", + "bytes": 8489915, + "allocated_bytes": 8491008, + "sha256": "3d007c36d1d79743f39940aec60b8908314dc8baae9e71fc7e93fe6148cc16bd", + "device": 27, + "inode": 1328659, + "mtime_ns": 1791921283502393950, + "ctime_ns": 1791921283502393950, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/wire.bin", + "run_id": "3a0e69f8e71c4c7995d21c8157edc230" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/fields.zip", + "bytes": 17023877, + "allocated_bytes": 17027072, + "sha256": "b591b00b9b9c32f3fe595d03197881f7626adeb89db2cacb2712900f52efe0c0", + "device": 27, + "inode": 1330226, + "mtime_ns": 1791921289250310282, + "ctime_ns": 1791921289250310282, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/fields.zip", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330274, + "mtime_ns": 1791921289558305911, + "ctime_ns": 1791921289558305911, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/wire.bin", + "bytes": 8489922, + "allocated_bytes": 8491008, + "sha256": "c583e7b9a00b057c1b2dea6f23dbf41c3770fc690a5e7430d8fe8558c733b481", + "device": 27, + "inode": 1330242, + "mtime_ns": 1791921289338309033, + "ctime_ns": 1791921289338309033, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/wire.bin", + "run_id": "48dbf653056a4d45ae53c6842e7d3aa3" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "334df086cd8cd6dc9689cf5faafd3631a2b0967b0b216c8a507c57083c2f5dee", + "device": 27, + "inode": 1330738, + "mtime_ns": 1791921295598219943, + "ctime_ns": 1791921295598219943, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/fields.zip", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330786, + "mtime_ns": 1791921295946214935, + "ctime_ns": 1791921295946214935, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "5dfafc4343e3d6e1b112bc0ca572d30ff129c0888288e5fc1a04836c824c127d", + "device": 27, + "inode": 1330754, + "mtime_ns": 1791921295698218504, + "ctime_ns": 1791921295698218504, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/wire.bin", + "run_id": "ecd6685cbd034a2e9c4c870b68ddd3a4" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/fields.zip", + "bytes": 17023761, + "allocated_bytes": 17027072, + "sha256": "ea5653a7f628cf292b2bb8c86737e900d4dc711eb197eca8683a5a8344ea03b2", + "device": 27, + "inode": 1331250, + "mtime_ns": 1791921301874130359, + "ctime_ns": 1791921301874130359, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/fields.zip", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331298, + "mtime_ns": 1791921302346123646, + "ctime_ns": 1791921302346123646, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/wire.bin", + "bytes": 8489864, + "allocated_bytes": 8491008, + "sha256": "b71ecbad87f3ccdf3ddc10079db639a91e5990075c32e73a427ab741968a61b6", + "device": 27, + "inode": 1331266, + "mtime_ns": 1791921302214125523, + "ctime_ns": 1791921302214125523, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/wire.bin", + "run_id": "8a22050ad7fc496db53e67d091de062a" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/fields.zip", + "bytes": 17023801, + "allocated_bytes": 17027072, + "sha256": "64620a51df29a223938c6db43a2450296e412e60dff944ba128df5f8e3df8605", + "device": 27, + "inode": 1331746, + "mtime_ns": 1791921308806031746, + "ctime_ns": 1791921308806031746, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/fields.zip", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331794, + "mtime_ns": 1791921309166026625, + "ctime_ns": 1791921309166026625, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/wire.bin", + "bytes": 8489885, + "allocated_bytes": 8491008, + "sha256": "737208512c9f32591d0bd125b2c5eab871827f395af6e5ce10eeb6a03c9a82dd", + "device": 27, + "inode": 1331762, + "mtime_ns": 1791921308930029982, + "ctime_ns": 1791921308930029982, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/wire.bin", + "run_id": "8d2985a122b644fa96f3bafd3146edf5" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/fields.zip", + "bytes": 17023919, + "allocated_bytes": 17027072, + "sha256": "c0b9c23b226ca261393aa741358ea3c03e476ce80bfae1dc956852730cee2948", + "device": 27, + "inode": 1332258, + "mtime_ns": 1791921315389937701, + "ctime_ns": 1791921315389937701, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/fields.zip", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1332306, + "mtime_ns": 1791921315813931667, + "ctime_ns": 1791921315813931667, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/wire.bin", + "bytes": 8489943, + "allocated_bytes": 8491008, + "sha256": "ca94a63ef1a73229218199860a5391e858e766874ebc1b7362a06f65228d4aad", + "device": 27, + "inode": 1332274, + "mtime_ns": 1791921315493936221, + "ctime_ns": 1791921315493936221, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/wire.bin", + "run_id": "c3f7594d311843ea8487427556c84810" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "7ad4b8d7e5de6b4a3aec1539062661df7551f74f66f0fe92285802dad3e99cd9", + "device": 27, + "inode": 1327956, + "mtime_ns": 1791921322365836792, + "ctime_ns": 1791921322365836792, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/fields.zip", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328003, + "mtime_ns": 1791921322837830039, + "ctime_ns": 1791921322837830039, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "6cc696471fe628586099d19f15b2202093ffebdcd08e6b0c27663473b0cc8a51", + "device": 27, + "inode": 1327972, + "mtime_ns": 1791921322501834824, + "ctime_ns": 1791921322501834824, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/wire.bin", + "run_id": "e46bc5a79331428580dba01bb671d25b" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/fields.zip", + "bytes": 17023839, + "allocated_bytes": 17027072, + "sha256": "c9ceee9e50861ee589b3e1c324d5dcb187cf4f79322fd12d208e450e86544cc1", + "device": 27, + "inode": 1328331, + "mtime_ns": 1791921329705731408, + "ctime_ns": 1791921329705731408, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/fields.zip", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328375, + "mtime_ns": 1791921330261723351, + "ctime_ns": 1791921330261723351, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/wire.bin", + "bytes": 8489903, + "allocated_bytes": 8491008, + "sha256": "fff88c99b9d04ebf30440e66155fe4e750083295989bca635ab730e6498f64f2", + "device": 27, + "inode": 1328342, + "mtime_ns": 1791921330129725264, + "ctime_ns": 1791921330129725264, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/wire.bin", + "run_id": "cea0a01e08c4488cbdba92f1662c7020" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/fields.zip", + "bytes": 17023819, + "allocated_bytes": 17027072, + "sha256": "3043098224dc4fedf7ba737e0238b785fa136c02816ee1297a00ccd6524680bc", + "device": 27, + "inode": 1328571, + "mtime_ns": 1791921337821615079, + "ctime_ns": 1791921337821615079, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/fields.zip", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328613, + "mtime_ns": 1791921338301608202, + "ctime_ns": 1791921338301608202, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/wire.bin", + "bytes": 8489893, + "allocated_bytes": 8491008, + "sha256": "db690f103de97e19d2de68beebe6d735415a8216a21f1c35559634944cd0802a", + "device": 27, + "inode": 1328582, + "mtime_ns": 1791921337913613761, + "ctime_ns": 1791921337913613761, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/wire.bin", + "run_id": "80ad7f8474ae4e9384483b9463206ab2" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/fields.zip", + "bytes": 17023745, + "allocated_bytes": 17027072, + "sha256": "93e4dd14ea507948ab329a08ae86282d4548422e59fa88d2644e439f1138db1d", + "device": 27, + "inode": 1332690, + "mtime_ns": 1791921346685488307, + "ctime_ns": 1791921346685488307, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/fields.zip", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1332738, + "mtime_ns": 1791921347177481284, + "ctime_ns": 1791921347177481284, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/wire.bin", + "bytes": 8489856, + "allocated_bytes": 8491008, + "sha256": "8fe7fd42d00fd96007941a853d1cacd6324b626e3b090cab4d86c7078ef8a713", + "device": 27, + "inode": 1332706, + "mtime_ns": 1791921346813486479, + "ctime_ns": 1791921346813486479, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/wire.bin", + "run_id": "d1700cfb1fb249e3b627a9cb1daadcec" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/fields.zip", + "bytes": 17023869, + "allocated_bytes": 17027072, + "sha256": "54f44df2fe839f283511d125a5000138070243552504df883b349455d63e46c3", + "device": 27, + "inode": 1333122, + "mtime_ns": 1791921354021383449, + "ctime_ns": 1791921354021383449, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/fields.zip", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333170, + "mtime_ns": 1791921354501376578, + "ctime_ns": 1791921354501376578, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/wire.bin", + "bytes": 8489919, + "allocated_bytes": 8491008, + "sha256": "d0fc71906b63ed46038380f1ac09f211e48b4a44d4ce86dfd47210f9eaa2c259", + "device": 27, + "inode": 1333138, + "mtime_ns": 1791921354121382018, + "ctime_ns": 1791921354121382018, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/wire.bin", + "run_id": "cf94711d8a2047c08a64c0219f474017" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/fields.zip", + "bytes": 17023873, + "allocated_bytes": 17027072, + "sha256": "dcd360f9f6c68c557e610385745f1f036447848aadb27126a68b729d273d667e", + "device": 27, + "inode": 1333330, + "mtime_ns": 1791921361581274307, + "ctime_ns": 1791921361581274307, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/fields.zip", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333378, + "mtime_ns": 1791921362061267293, + "ctime_ns": 1791921362061267293, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/wire.bin", + "bytes": 8489920, + "allocated_bytes": 8491008, + "sha256": "886d70ea42c70216ab285fde707b765c2fa81eeb1a10b4f4d5f10c371bb71bb4", + "device": 27, + "inode": 1333346, + "mtime_ns": 1791921361693272671, + "ctime_ns": 1791921361693272671, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/wire.bin", + "run_id": "4a9a129e8405405ab12b866edd766e48" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-iREMrq/generation/100/fields.zip", + "bytes": 17023815, + "allocated_bytes": 17027072, + "sha256": "d801973641c7ebd42b193c37b9c06053d0db59a4ab34dcfd7fcddf8a82d53600", + "device": 27, + "inode": 1328403, + "mtime_ns": 1791921259246749697, + "ctime_ns": 1791921259246749697, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/fields.zip", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-iREMrq/generation/100/wire.bin", + "bytes": 8489891, + "allocated_bytes": 8491008, + "sha256": "851969c2e6cfe8782078247271a13b950dcc26e48201a6e3af76bb03b844e4b6", + "device": 27, + "inode": 1328419, + "mtime_ns": 1791921259326748597, + "ctime_ns": 1791921259326748597, + "category": "qa_generation_payload", + "authoritative_path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/wire.bin", + "run_id": "df53534ae6154ec8a4788f8378a8f004" + } + ], + "removable_empty_output_directories": [ + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output", + "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output" + ], + "verified_completed_runs": { + "4a9a129e8405405ab12b866edd766e48": { + "state": "completed", + "custody_receipt_sha256": "a95a4aad6d5f6d015f1b022d50c088f85d2fd3a918d00b22944776315131cfe2", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333283, + "mtime_ns": 1791921360681287460, + "ctime_ns": 1791921360681287460 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/fields.zip", + "bytes": 17023873, + "allocated_bytes": 17027072, + "sha256": "dcd360f9f6c68c557e610385745f1f036447848aadb27126a68b729d273d667e", + "device": 27, + "inode": 1333253, + "mtime_ns": 1791921360698477183, + "ctime_ns": 1791921360698477183 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/producer-receipt.json", + "bytes": 4242, + "allocated_bytes": 8192, + "sha256": "7243d456b751863acc9b0e69525ffa255d473d609d7c0b2f1814805fa2f28849", + "device": 27, + "inode": 1333254, + "mtime_ns": 1791921360720627722, + "ctime_ns": 1791921360720627722 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/4a9a129e8405405ab12b866edd766e48/wire.bin", + "bytes": 8489920, + "allocated_bytes": 8491008, + "sha256": "886d70ea42c70216ab285fde707b765c2fa81eeb1a10b4f4d5f10c371bb71bb4", + "device": 27, + "inode": 1333255, + "mtime_ns": 1791921360727297628, + "ctime_ns": 1791921360727297628 + } + } + }, + "48dbf653056a4d45ae53c6842e7d3aa3": { + "state": "completed", + "custody_receipt_sha256": "db4a2bab92b8029086314aab067f09bb25c79c6065594f3618aedc81c69d5fb7", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330114, + "mtime_ns": 1791921288766317150, + "ctime_ns": 1791921288766317150 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/fields.zip", + "bytes": 17023877, + "allocated_bytes": 17027072, + "sha256": "b591b00b9b9c32f3fe595d03197881f7626adeb89db2cacb2712900f52efe0c0", + "device": 27, + "inode": 1330130, + "mtime_ns": 1791921288781802404, + "ctime_ns": 1791921288781802404 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/producer-receipt.json", + "bytes": 4244, + "allocated_bytes": 8192, + "sha256": "f52d752fc048b5708208e6b75565ff01b0a0d9943c84f7a2391497090892bc24", + "device": 27, + "inode": 1330146, + "mtime_ns": 1791921288788067096, + "ctime_ns": 1791921288788067096 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/48dbf653056a4d45ae53c6842e7d3aa3/wire.bin", + "bytes": 8489922, + "allocated_bytes": 8491008, + "sha256": "c583e7b9a00b057c1b2dea6f23dbf41c3770fc690a5e7430d8fe8558c733b481", + "device": 27, + "inode": 1330162, + "mtime_ns": 1791921288794780088, + "ctime_ns": 1791921288794780088 + } + } + }, + "cea0a01e08c4488cbdba92f1662c7020": { + "state": "completed", + "custody_receipt_sha256": "be236071a119a382f653211ac4d630046b39f280665a253636b3808bae94c626", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328277, + "mtime_ns": 1791921328785744741, + "ctime_ns": 1791921328785744741 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/fields.zip", + "bytes": 17023839, + "allocated_bytes": 17027072, + "sha256": "c9ceee9e50861ee589b3e1c324d5dcb187cf4f79322fd12d208e450e86544cc1", + "device": 27, + "inode": 1328246, + "mtime_ns": 1791921328799408428, + "ctime_ns": 1791921328799408428 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/producer-receipt.json", + "bytes": 4225, + "allocated_bytes": 8192, + "sha256": "8fde547aed7066c30f708e555e1d94c6685d68c8972d8e4926a2201d0f4344df", + "device": 27, + "inode": 1328247, + "mtime_ns": 1791921328885485282, + "ctime_ns": 1791921328885485282 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cea0a01e08c4488cbdba92f1662c7020/wire.bin", + "bytes": 8489903, + "allocated_bytes": 8491008, + "sha256": "fff88c99b9d04ebf30440e66155fe4e750083295989bca635ab730e6498f64f2", + "device": 27, + "inode": 1328248, + "mtime_ns": 1791921328892819725, + "ctime_ns": 1791921328892819725 + } + } + }, + "ecd6685cbd034a2e9c4c870b68ddd3a4": { + "state": "completed", + "custody_receipt_sha256": "21c126286f26937fb8177fbbd6d2795cea693514e574963aa1c458a2b3970c24", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1330626, + "mtime_ns": 1791921295050227832, + "ctime_ns": 1791921295050227832 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "334df086cd8cd6dc9689cf5faafd3631a2b0967b0b216c8a507c57083c2f5dee", + "device": 27, + "inode": 1330642, + "mtime_ns": 1791921295065395839, + "ctime_ns": 1791921295065395839 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/producer-receipt.json", + "bytes": 4233, + "allocated_bytes": 8192, + "sha256": "eac0ba8fe582c5f83bd0973acca7be44068e2e799c74340f34764bd421bf2924", + "device": 27, + "inode": 1330658, + "mtime_ns": 1791921295077146146, + "ctime_ns": 1791921295077146146 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/ecd6685cbd034a2e9c4c870b68ddd3a4/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "5dfafc4343e3d6e1b112bc0ca572d30ff129c0888288e5fc1a04836c824c127d", + "device": 27, + "inode": 1330674, + "mtime_ns": 1791921295083766820, + "ctime_ns": 1791921295083766820 + } + } + }, + "44272474e32544898feb39f651d997b9": { + "state": "completed", + "custody_receipt_sha256": "e6c2876fb0faf91b7abb02ef7f27ae63afae9818a7d40790c43e179405d42586", + "drained": true, + "files": { + "result.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/result.json", + "bytes": 5067, + "allocated_bytes": 8192, + "sha256": "62ccabd6ca01966992665573fbac85c608c9dfbd358ff4e1e6715e50688e4394", + "device": 27, + "inode": 1211618, + "mtime_ns": 1791919168449318706, + "ctime_ns": 1791919168449318706 + }, + "checkpoint.pt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/checkpoint.pt", + "bytes": 2874, + "allocated_bytes": 4096, + "sha256": "7d44cfb241c67e0d2b5349c5dbf5b0251a4ea6f4bc3f5f25177c4ecfc6e6911b", + "device": 27, + "inode": 1211603, + "mtime_ns": 1791919168450769418, + "ctime_ns": 1791919168450769418 + }, + "array.npy": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/44272474e32544898feb39f651d997b9/array.npy", + "bytes": 224, + "allocated_bytes": 4096, + "sha256": "d4527f6b3061eb636796c8343fa55690843b423063c32c4506be611a678d9fc2", + "device": 27, + "inode": 1211604, + "mtime_ns": 1791919168451456988, + "ctime_ns": 1791919168451456988 + } + } + }, + "8a22050ad7fc496db53e67d091de062a": { + "state": "completed", + "custody_receipt_sha256": "17a2068b9803acbb2ba9d5862a2b2e925fe636586732461f28ecb30f63224d29", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331138, + "mtime_ns": 1791921301226139575, + "ctime_ns": 1791921301226139575 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/fields.zip", + "bytes": 17023761, + "allocated_bytes": 17027072, + "sha256": "ea5653a7f628cf292b2bb8c86737e900d4dc711eb197eca8683a5a8344ea03b2", + "device": 27, + "inode": 1331154, + "mtime_ns": 1791921301243502876, + "ctime_ns": 1791921301243502876 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/producer-receipt.json", + "bytes": 4186, + "allocated_bytes": 8192, + "sha256": "67352c1ca8559250bbc57b408bc8f32517fc0dd173307ac53f0a155e1e73b187", + "device": 27, + "inode": 1331170, + "mtime_ns": 1791921301279474578, + "ctime_ns": 1791921301279474578 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8a22050ad7fc496db53e67d091de062a/wire.bin", + "bytes": 8489864, + "allocated_bytes": 8491008, + "sha256": "b71ecbad87f3ccdf3ddc10079db639a91e5990075c32e73a427ab741968a61b6", + "device": 27, + "inode": 1331186, + "mtime_ns": 1791921301286330729, + "ctime_ns": 1791921301286330729 + } + } + }, + "80ad7f8474ae4e9384483b9463206ab2": { + "state": "completed", + "custody_receipt_sha256": "9f492922ed705955eecd1dc08992f20431f5eb04bfa2b5d5503b417bf059310e", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328501, + "mtime_ns": 1791921336953627514, + "ctime_ns": 1791921336953627514 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/fields.zip", + "bytes": 17023819, + "allocated_bytes": 17027072, + "sha256": "3043098224dc4fedf7ba737e0238b785fa136c02816ee1297a00ccd6524680bc", + "device": 27, + "inode": 1328455, + "mtime_ns": 1791921336973766729, + "ctime_ns": 1791921336973766729 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/producer-receipt.json", + "bytes": 4215, + "allocated_bytes": 8192, + "sha256": "8f1f0b8d51cdbc2e1e14609fd75fca5c47451f22e6afddac0c9756456bf5c97c", + "device": 27, + "inode": 1328456, + "mtime_ns": 1791921337014414393, + "ctime_ns": 1791921337014414393 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/80ad7f8474ae4e9384483b9463206ab2/wire.bin", + "bytes": 8489893, + "allocated_bytes": 8491008, + "sha256": "db690f103de97e19d2de68beebe6d735415a8216a21f1c35559634944cd0802a", + "device": 27, + "inode": 1328457, + "mtime_ns": 1791921337022106814, + "ctime_ns": 1791921337022106814 + } + } + }, + "0cc3ef6b5f9e460f873c8eda1ba3d525": { + "state": "completed", + "custody_receipt_sha256": "89d1062ee70ea2a47eb1006b271280d0e8e48fb567b6b0c52be4dd68ff649318", + "drained": true, + "files": { + "result.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/result.json", + "bytes": 1007942, + "allocated_bytes": 1011712, + "sha256": "d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448", + "device": 27, + "inode": 1211470, + "mtime_ns": 1791919934346802314, + "ctime_ns": 1791919934346802314 + }, + "timing.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/timing.json", + "bytes": 871725, + "allocated_bytes": 872448, + "sha256": "0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b", + "device": 27, + "inode": 1211471, + "mtime_ns": 1791919934349141842, + "ctime_ns": 1791919934349141842 + }, + "input-model.pt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/input-model.pt", + "bytes": 272149, + "allocated_bytes": 274432, + "sha256": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "device": 27, + "inode": 1211472, + "mtime_ns": 1791919934350369316, + "ctime_ns": 1791919934350369316 + }, + "comparison-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/comparison-receipt.json", + "bytes": 26712, + "allocated_bytes": 28672, + "sha256": "d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a", + "device": 27, + "inode": 1211475, + "mtime_ns": 1791919934351077817, + "ctime_ns": 1791919934351077817 + }, + "input-freeze.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/input-freeze.json", + "bytes": 96833, + "allocated_bytes": 98304, + "sha256": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "device": 27, + "inode": 1211476, + "mtime_ns": 1791919934351702145, + "ctime_ns": 1791919934351702145 + }, + "comparison-freeze.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/0cc3ef6b5f9e460f873c8eda1ba3d525/comparison-freeze.json", + "bytes": 1400, + "allocated_bytes": 4096, + "sha256": "5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076", + "device": 27, + "inode": 1211477, + "mtime_ns": 1791919934352312051, + "ctime_ns": 1791919934352312051 + } + } + }, + "8d2985a122b644fa96f3bafd3146edf5": { + "state": "completed", + "custody_receipt_sha256": "8f3b80806e38efdb9655b19ba5aef1ff026763558f8886e8dd2d7dc30b5371ff", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1331634, + "mtime_ns": 1791921308338038404, + "ctime_ns": 1791921308338038404 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/fields.zip", + "bytes": 17023801, + "allocated_bytes": 17027072, + "sha256": "64620a51df29a223938c6db43a2450296e412e60dff944ba128df5f8e3df8605", + "device": 27, + "inode": 1331650, + "mtime_ns": 1791921308354464685, + "ctime_ns": 1791921308354464685 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/producer-receipt.json", + "bytes": 4205, + "allocated_bytes": 8192, + "sha256": "f6abe10655bfd3ac050b5755c5c745d6b403983592d8f5905952dead31b82244", + "device": 27, + "inode": 1331666, + "mtime_ns": 1791921308363743129, + "ctime_ns": 1791921308363743129 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/8d2985a122b644fa96f3bafd3146edf5/wire.bin", + "bytes": 8489885, + "allocated_bytes": 8491008, + "sha256": "737208512c9f32591d0bd125b2c5eab871827f395af6e5ce10eeb6a03c9a82dd", + "device": 27, + "inode": 1331682, + "mtime_ns": 1791921308378953857, + "ctime_ns": 1791921308378953857 + } + } + }, + "df53534ae6154ec8a4788f8378a8f004": { + "state": "completed", + "custody_receipt_sha256": "f2e4e7ceb4f71c47bfa48a212668ac4ba698eeb44cd9337acd92836684e810d6", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328356, + "mtime_ns": 1791921258822755519, + "ctime_ns": 1791921258822755519 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/fields.zip", + "bytes": 17023815, + "allocated_bytes": 17027072, + "sha256": "d801973641c7ebd42b193c37b9c06053d0db59a4ab34dcfd7fcddf8a82d53600", + "device": 27, + "inode": 1328326, + "mtime_ns": 1791921258839988208, + "ctime_ns": 1791921258839988208 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/producer-receipt.json", + "bytes": 4213, + "allocated_bytes": 8192, + "sha256": "9019802f4a742730e9b63a09718480af30cd3ecbb87924938886d10439ecfca5", + "device": 27, + "inode": 1328327, + "mtime_ns": 1791921258845481879, + "ctime_ns": 1791921258845481879 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/df53534ae6154ec8a4788f8378a8f004/wire.bin", + "bytes": 8489891, + "allocated_bytes": 8491008, + "sha256": "851969c2e6cfe8782078247271a13b950dcc26e48201a6e3af76bb03b844e4b6", + "device": 27, + "inode": 1328328, + "mtime_ns": 1791921258851907892, + "ctime_ns": 1791921258851907892 + } + } + }, + "d1700cfb1fb249e3b627a9cb1daadcec": { + "state": "completed", + "custody_receipt_sha256": "fa5637f5931b4be3da41dd212fba7a78ea0d701ccdcfd6fe884c9927297dfb74", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1332658, + "mtime_ns": 1791921345869499954, + "ctime_ns": 1791921345869499954 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/fields.zip", + "bytes": 17023745, + "allocated_bytes": 17027072, + "sha256": "93e4dd14ea507948ab329a08ae86282d4548422e59fa88d2644e439f1138db1d", + "device": 27, + "inode": 1328695, + "mtime_ns": 1791921345888273836, + "ctime_ns": 1791921345888273836 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/producer-receipt.json", + "bytes": 4178, + "allocated_bytes": 8192, + "sha256": "377b8a1c150f9c11ab1a41f0b7c9d80cc6913c5c861c0951e17d83a7cedf0164", + "device": 27, + "inode": 1328696, + "mtime_ns": 1791921345907197898, + "ctime_ns": 1791921345907197898 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/d1700cfb1fb249e3b627a9cb1daadcec/wire.bin", + "bytes": 8489856, + "allocated_bytes": 8491008, + "sha256": "8fe7fd42d00fd96007941a853d1cacd6324b626e3b090cab4d86c7078ef8a713", + "device": 27, + "inode": 1328697, + "mtime_ns": 1791921345914033529, + "ctime_ns": 1791921345914033529 + } + } + }, + "3a0e69f8e71c4c7995d21c8157edc230": { + "state": "completed", + "custody_receipt_sha256": "19bdeea8e9840c0b2b1f330370615b4c3d4b20423e846c509d53ff6561572ad9", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1328596, + "mtime_ns": 1791921282670405872, + "ctime_ns": 1791921282670405872 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/fields.zip", + "bytes": 17023863, + "allocated_bytes": 17027072, + "sha256": "1c137295d187099956c0b96149e0cd1babaab893d2d7396e563593d8abc85028", + "device": 27, + "inode": 1328566, + "mtime_ns": 1791921282686678480, + "ctime_ns": 1791921282686678480 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/producer-receipt.json", + "bytes": 4237, + "allocated_bytes": 8192, + "sha256": "af4d5a0e2b42bdd62b05db96bdc4cd8b9d85bbdd0028bb8b23a3dac387a5080d", + "device": 27, + "inode": 1328567, + "mtime_ns": 1791921282692318462, + "ctime_ns": 1791921282692318462 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/3a0e69f8e71c4c7995d21c8157edc230/wire.bin", + "bytes": 8489915, + "allocated_bytes": 8491008, + "sha256": "3d007c36d1d79743f39940aec60b8908314dc8baae9e71fc7e93fe6148cc16bd", + "device": 27, + "inode": 1328568, + "mtime_ns": 1791921282698808645, + "ctime_ns": 1791921282698808645 + } + } + }, + "c3f7594d311843ea8487427556c84810": { + "state": "completed", + "custody_receipt_sha256": "373648c1254a64209afe8dbad3dbaf2a2c53970a43aadef62f31151ead13025e", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1332146, + "mtime_ns": 1791921314725947151, + "ctime_ns": 1791921314725947151 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/fields.zip", + "bytes": 17023919, + "allocated_bytes": 17027072, + "sha256": "c0b9c23b226ca261393aa741358ea3c03e476ce80bfae1dc956852730cee2948", + "device": 27, + "inode": 1332162, + "mtime_ns": 1791921314739845468, + "ctime_ns": 1791921314739845468 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/producer-receipt.json", + "bytes": 4265, + "allocated_bytes": 8192, + "sha256": "4152d0d691ccc78099b048081059a2b2608143e317ea6a0b322c2261fe366130", + "device": 27, + "inode": 1332178, + "mtime_ns": 1791921314760978174, + "ctime_ns": 1791921314760978174 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/c3f7594d311843ea8487427556c84810/wire.bin", + "bytes": 8489943, + "allocated_bytes": 8491008, + "sha256": "ca94a63ef1a73229218199860a5391e858e766874ebc1b7362a06f65228d4aad", + "device": 27, + "inode": 1332194, + "mtime_ns": 1791921314767838101, + "ctime_ns": 1791921314767838101 + } + } + }, + "11fc2be1aada490f9b67c11c190728f6": { + "state": "completed", + "custody_receipt_sha256": "82e0cb390d4f9264d31ec82bdb7a474dded0ee20eaa3ebeef2227ca663e41f65", + "drained": true, + "files": { + "result.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/result.json", + "bytes": 129795, + "allocated_bytes": 131072, + "sha256": "9b680ad1780fe276b5385805a06dcc3acb365f6b3475481af20e982e6495aa79", + "device": 27, + "inode": 1328365, + "mtime_ns": 1791921789308744829, + "ctime_ns": 1791921789308744829 + }, + "model.pt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/model.pt", + "bytes": 272149, + "allocated_bytes": 274432, + "sha256": "e08e85d991d2734383d14501d6dcfdaccb45dfd53370d883c26b44e84255f5f8", + "device": 27, + "inode": 1328366, + "mtime_ns": 1791921789310260432, + "ctime_ns": 1791921789310260432 + }, + "training.npz": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/training.npz", + "bytes": 50332156, + "allocated_bytes": 50335744, + "sha256": "3ca87734e4ebe41320328b130908dc88b85967dcc745f80f8c207cf29b539184", + "device": 27, + "inode": 1328367, + "mtime_ns": 1791921789359223061, + "ctime_ns": 1791921789359223061 + }, + "freeze.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/freeze.json", + "bytes": 79042, + "allocated_bytes": 81920, + "sha256": "020ac431def0540edca65b6009ee77ebf8a5eff414efd9cf7da38532ef711792", + "device": 27, + "inode": 1328368, + "mtime_ns": 1791921789385062161, + "ctime_ns": 1791921789385062161 + }, + "result-summary.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/result-summary.json", + "bytes": 611, + "allocated_bytes": 4096, + "sha256": "ad89e6106e83a06a6ac68dd0eb950338cf05ffba768f3fd72558cb03348188ec", + "device": 27, + "inode": 1328376, + "mtime_ns": 1791921789385930087, + "ctime_ns": 1791921789385930087 + }, + "input-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/11fc2be1aada490f9b67c11c190728f6/input-receipt.json", + "bytes": 55329, + "allocated_bytes": 57344, + "sha256": "1a8ec140ce18fee118ba9821353a2538b3caf8ab89b73a678bd8acdf03bed189", + "device": 27, + "inode": 1328377, + "mtime_ns": 1791921789386608524, + "ctime_ns": 1791921789386608524 + } + } + }, + "e46bc5a79331428580dba01bb671d25b": { + "state": "completed", + "custody_receipt_sha256": "f39a20d413ea338517c6faa1c63587d81b53489d5da5ff6e87b7538752b9ba4a", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1327924, + "mtime_ns": 1791921321593848095, + "ctime_ns": 1791921321593848095 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/fields.zip", + "bytes": 17023855, + "allocated_bytes": 17027072, + "sha256": "7ad4b8d7e5de6b4a3aec1539062661df7551f74f66f0fe92285802dad3e99cd9", + "device": 27, + "inode": 1327925, + "mtime_ns": 1791921321609719335, + "ctime_ns": 1791921321609719335 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/producer-receipt.json", + "bytes": 4233, + "allocated_bytes": 8192, + "sha256": "412a4936341eb2e6f0555135707fbc5fe37bba218c30d8132b30c80f84cf904a", + "device": 27, + "inode": 1327926, + "mtime_ns": 1791921321654285381, + "ctime_ns": 1791921321654285381 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/e46bc5a79331428580dba01bb671d25b/wire.bin", + "bytes": 8489911, + "allocated_bytes": 8491008, + "sha256": "6cc696471fe628586099d19f15b2202093ffebdcd08e6b0c27663473b0cc8a51", + "device": 27, + "inode": 1327927, + "mtime_ns": 1791921321660879555, + "ctime_ns": 1791921321660879555 + } + } + }, + "cf94711d8a2047c08a64c0219f474017": { + "state": "completed", + "custody_receipt_sha256": "7b2b54952f0de2c5fcd66d5b1735b3a2ce8cfed4acea687b7c383e8997f2259e", + "drained": true, + "files": { + "native-diagnostics.txt": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/native-diagnostics.txt", + "bytes": 0, + "allocated_bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "device": 27, + "inode": 1333090, + "mtime_ns": 1791921353221394900, + "ctime_ns": 1791921353221394900 + }, + "fields.zip": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/fields.zip", + "bytes": 17023869, + "allocated_bytes": 17027072, + "sha256": "54f44df2fe839f283511d125a5000138070243552504df883b349455d63e46c3", + "device": 27, + "inode": 1333075, + "mtime_ns": 1791921353238515118, + "ctime_ns": 1791921353238515118 + }, + "producer-receipt.json": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/producer-receipt.json", + "bytes": 4239, + "allocated_bytes": 8192, + "sha256": "18a5c3e99bff019f6839e721180e5fa25c13ce97369f3993721dc3910aef597a", + "device": 27, + "inode": 1333076, + "mtime_ns": 1791921353245654686, + "ctime_ns": 1791921353245654686 + }, + "wire.bin": { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/cf94711d8a2047c08a64c0219f474017/wire.bin", + "bytes": 8489919, + "allocated_bytes": 8491008, + "sha256": "d0fc71906b63ed46038380f1ac09f211e48b4a44d4ce86dfd47210f9eaa2c259", + "device": 27, + "inode": 1333077, + "mtime_ns": 1791921353252945847, + "ctime_ns": 1791921353252945847 + } + } + } + }, + "excluded_runs": [ + { + "id": "28a3dd42f3e64fab89136da1d7e5d8a0", + "state": "failed" + }, + { + "id": "ab46bb3b6e2944a0a0794776da0352f7", + "state": "failed" + }, + { + "id": "a8298e4e255a46d4beef0ec9f73e38d0", + "state": "cancelled" + }, + { + "id": "ece8abe0a4024d1483048d91b7980cf0", + "state": "failed" + } + ], + "preserve": [ + "all failed/active Run directories", + "source/input/spec/custody staging", + "published artifacts", + "managed releases", + "runtime snapshots", + "QA cold reports, receipts, screenshots and prepared inventory" + ], + "logical_bytes": 921437763, + "allocated_bytes": 921657344 +} diff --git a/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-receipt.json b/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-receipt.json new file mode 100644 index 00000000..db65dc61 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5c/duplicate-cleanup-receipt.json @@ -0,0 +1,768 @@ +{ + "inventory_sha256": "973ba68f29ea965453765b4d53e9f59f8bcdc0054c4581ced290ac23d9292f2b", + "outcomes": [ + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/fields.zip", + "outcome": "unlinked", + "sha256": "dcd360f9f6c68c557e610385745f1f036447848aadb27126a68b729d273d667e", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "7243d456b751863acc9b0e69525ffa255d473d609d7c0b2f1814805fa2f28849", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output/wire.bin", + "outcome": "unlinked", + "sha256": "886d70ea42c70216ab285fde707b765c2fa81eeb1a10b4f4d5f10c371bb71bb4", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/fields.zip", + "outcome": "unlinked", + "sha256": "b591b00b9b9c32f3fe595d03197881f7626adeb89db2cacb2712900f52efe0c0", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "f52d752fc048b5708208e6b75565ff01b0a0d9943c84f7a2391497090892bc24", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output/wire.bin", + "outcome": "unlinked", + "sha256": "c583e7b9a00b057c1b2dea6f23dbf41c3770fc690a5e7430d8fe8558c733b481", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/fields.zip", + "outcome": "unlinked", + "sha256": "c9ceee9e50861ee589b3e1c324d5dcb187cf4f79322fd12d208e450e86544cc1", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "8fde547aed7066c30f708e555e1d94c6685d68c8972d8e4926a2201d0f4344df", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output/wire.bin", + "outcome": "unlinked", + "sha256": "fff88c99b9d04ebf30440e66155fe4e750083295989bca635ab730e6498f64f2", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/fields.zip", + "outcome": "unlinked", + "sha256": "334df086cd8cd6dc9689cf5faafd3631a2b0967b0b216c8a507c57083c2f5dee", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "eac0ba8fe582c5f83bd0973acca7be44068e2e799c74340f34764bd421bf2924", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output/wire.bin", + "outcome": "unlinked", + "sha256": "5dfafc4343e3d6e1b112bc0ca572d30ff129c0888288e5fc1a04836c824c127d", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/result.json", + "outcome": "unlinked", + "sha256": "62ccabd6ca01966992665573fbac85c608c9dfbd358ff4e1e6715e50688e4394", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/checkpoint.pt", + "outcome": "unlinked", + "sha256": "7d44cfb241c67e0d2b5349c5dbf5b0251a4ea6f4bc3f5f25177c4ecfc6e6911b", + "allocated_bytes": 4096 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output/array.npy", + "outcome": "unlinked", + "sha256": "d4527f6b3061eb636796c8343fa55690843b423063c32c4506be611a678d9fc2", + "allocated_bytes": 4096 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/fields.zip", + "outcome": "unlinked", + "sha256": "ea5653a7f628cf292b2bb8c86737e900d4dc711eb197eca8683a5a8344ea03b2", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "67352c1ca8559250bbc57b408bc8f32517fc0dd173307ac53f0a155e1e73b187", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output/wire.bin", + "outcome": "unlinked", + "sha256": "b71ecbad87f3ccdf3ddc10079db639a91e5990075c32e73a427ab741968a61b6", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/fields.zip", + "outcome": "unlinked", + "sha256": "3043098224dc4fedf7ba737e0238b785fa136c02816ee1297a00ccd6524680bc", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "8f1f0b8d51cdbc2e1e14609fd75fca5c47451f22e6afddac0c9756456bf5c97c", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output/wire.bin", + "outcome": "unlinked", + "sha256": "db690f103de97e19d2de68beebe6d735415a8216a21f1c35559634944cd0802a", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/result.json", + "outcome": "unlinked", + "sha256": "d8cd19061749c53b54bd024a525ac319ebbe2330d8a08bdb62ae1511c2189448", + "allocated_bytes": 1011712 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/timing.json", + "outcome": "unlinked", + "sha256": "0ceda1dcd4d1faf29c20930e987b7b7dbd3f9ddd6c6015bf9986408ef9f78b1b", + "allocated_bytes": 872448 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/input-model.pt", + "outcome": "unlinked", + "sha256": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "allocated_bytes": 274432 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/comparison-receipt.json", + "outcome": "unlinked", + "sha256": "d5ccb93643716e5653753bb92434e26016131d705ca3e0ffe2967e010144216a", + "allocated_bytes": 28672 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/input-freeze.json", + "outcome": "unlinked", + "sha256": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "allocated_bytes": 98304 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output/comparison-freeze.json", + "outcome": "unlinked", + "sha256": "5bda3d3f9aaeeca9636c8e60307845c0fe483bdad66844b6959bb57d4d3f1076", + "allocated_bytes": 4096 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/fields.zip", + "outcome": "unlinked", + "sha256": "64620a51df29a223938c6db43a2450296e412e60dff944ba128df5f8e3df8605", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "f6abe10655bfd3ac050b5755c5c745d6b403983592d8f5905952dead31b82244", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output/wire.bin", + "outcome": "unlinked", + "sha256": "737208512c9f32591d0bd125b2c5eab871827f395af6e5ce10eeb6a03c9a82dd", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/fields.zip", + "outcome": "unlinked", + "sha256": "d801973641c7ebd42b193c37b9c06053d0db59a4ab34dcfd7fcddf8a82d53600", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "9019802f4a742730e9b63a09718480af30cd3ecbb87924938886d10439ecfca5", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output/wire.bin", + "outcome": "unlinked", + "sha256": "851969c2e6cfe8782078247271a13b950dcc26e48201a6e3af76bb03b844e4b6", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/fields.zip", + "outcome": "unlinked", + "sha256": "93e4dd14ea507948ab329a08ae86282d4548422e59fa88d2644e439f1138db1d", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "377b8a1c150f9c11ab1a41f0b7c9d80cc6913c5c861c0951e17d83a7cedf0164", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output/wire.bin", + "outcome": "unlinked", + "sha256": "8fe7fd42d00fd96007941a853d1cacd6324b626e3b090cab4d86c7078ef8a713", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/fields.zip", + "outcome": "unlinked", + "sha256": "1c137295d187099956c0b96149e0cd1babaab893d2d7396e563593d8abc85028", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "af4d5a0e2b42bdd62b05db96bdc4cd8b9d85bbdd0028bb8b23a3dac387a5080d", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output/wire.bin", + "outcome": "unlinked", + "sha256": "3d007c36d1d79743f39940aec60b8908314dc8baae9e71fc7e93fe6148cc16bd", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/fields.zip", + "outcome": "unlinked", + "sha256": "c0b9c23b226ca261393aa741358ea3c03e476ce80bfae1dc956852730cee2948", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "4152d0d691ccc78099b048081059a2b2608143e317ea6a0b322c2261fe366130", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output/wire.bin", + "outcome": "unlinked", + "sha256": "ca94a63ef1a73229218199860a5391e858e766874ebc1b7362a06f65228d4aad", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/result.json", + "outcome": "unlinked", + "sha256": "9b680ad1780fe276b5385805a06dcc3acb365f6b3475481af20e982e6495aa79", + "allocated_bytes": 131072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/model.pt", + "outcome": "unlinked", + "sha256": "e08e85d991d2734383d14501d6dcfdaccb45dfd53370d883c26b44e84255f5f8", + "allocated_bytes": 274432 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/training.npz", + "outcome": "unlinked", + "sha256": "3ca87734e4ebe41320328b130908dc88b85967dcc745f80f8c207cf29b539184", + "allocated_bytes": 50335744 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/freeze.json", + "outcome": "unlinked", + "sha256": "020ac431def0540edca65b6009ee77ebf8a5eff414efd9cf7da38532ef711792", + "allocated_bytes": 81920 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/result-summary.json", + "outcome": "unlinked", + "sha256": "ad89e6106e83a06a6ac68dd0eb950338cf05ffba768f3fd72558cb03348188ec", + "allocated_bytes": 4096 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output/input-receipt.json", + "outcome": "unlinked", + "sha256": "1a8ec140ce18fee118ba9821353a2538b3caf8ab89b73a678bd8acdf03bed189", + "allocated_bytes": 57344 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/fields.zip", + "outcome": "unlinked", + "sha256": "7ad4b8d7e5de6b4a3aec1539062661df7551f74f66f0fe92285802dad3e99cd9", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "412a4936341eb2e6f0555135707fbc5fe37bba218c30d8132b30c80f84cf904a", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output/wire.bin", + "outcome": "unlinked", + "sha256": "6cc696471fe628586099d19f15b2202093ffebdcd08e6b0c27663473b0cc8a51", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/fields.zip", + "outcome": "unlinked", + "sha256": "54f44df2fe839f283511d125a5000138070243552504df883b349455d63e46c3", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/producer-receipt.json", + "outcome": "unlinked", + "sha256": "18a5c3e99bff019f6839e721180e5fa25c13ce97369f3993721dc3910aef597a", + "allocated_bytes": 8192 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output/wire.bin", + "outcome": "unlinked", + "sha256": "d0fc71906b63ed46038380f1ac09f211e48b4a44d4ce86dfd47210f9eaa2c259", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-0.zip", + "outcome": "unlinked", + "sha256": "c21553baf38d2aa44a17eba441d039d9c6c22e863ec436c1e7e524ce923d0f14", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-1.zip", + "outcome": "unlinked", + "sha256": "e3dbff795de37df43501fc26010635242f600d3c3e228188bd2e63dbf17066b6", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-2.zip", + "outcome": "unlinked", + "sha256": "4e7bdc2b16554a6c73321bbd15329408a39f710a02ef34fc8b29507493bd8e0e", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-3.zip", + "outcome": "unlinked", + "sha256": "fd1e3412b8ce639ece6821f69f3981127d94052a88ab28649844d630b5038c4c", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-4.zip", + "outcome": "unlinked", + "sha256": "296f07e01ab0cf0dc428f9b680b0d02026705494847b9ee23845d46ce66ec893", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/training-pair-5.zip", + "outcome": "unlinked", + "sha256": "06e843c74f9cdda576a52859b736435eb6c8aee7f87b565de8dcc6625824c64b", + "allocated_bytes": 34148352 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/fields.zip", + "outcome": "unlinked", + "sha256": "1c137295d187099956c0b96149e0cd1babaab893d2d7396e563593d8abc85028", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/100/wire.bin", + "outcome": "unlinked", + "sha256": "3d007c36d1d79743f39940aec60b8908314dc8baae9e71fc7e93fe6148cc16bd", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/fields.zip", + "outcome": "unlinked", + "sha256": "b591b00b9b9c32f3fe595d03197881f7626adeb89db2cacb2712900f52efe0c0", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/101/wire.bin", + "outcome": "unlinked", + "sha256": "c583e7b9a00b057c1b2dea6f23dbf41c3770fc690a5e7430d8fe8558c733b481", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/fields.zip", + "outcome": "unlinked", + "sha256": "334df086cd8cd6dc9689cf5faafd3631a2b0967b0b216c8a507c57083c2f5dee", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/102/wire.bin", + "outcome": "unlinked", + "sha256": "5dfafc4343e3d6e1b112bc0ca572d30ff129c0888288e5fc1a04836c824c127d", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/fields.zip", + "outcome": "unlinked", + "sha256": "ea5653a7f628cf292b2bb8c86737e900d4dc711eb197eca8683a5a8344ea03b2", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/103/wire.bin", + "outcome": "unlinked", + "sha256": "b71ecbad87f3ccdf3ddc10079db639a91e5990075c32e73a427ab741968a61b6", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/fields.zip", + "outcome": "unlinked", + "sha256": "64620a51df29a223938c6db43a2450296e412e60dff944ba128df5f8e3df8605", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/104/wire.bin", + "outcome": "unlinked", + "sha256": "737208512c9f32591d0bd125b2c5eab871827f395af6e5ce10eeb6a03c9a82dd", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/fields.zip", + "outcome": "unlinked", + "sha256": "c0b9c23b226ca261393aa741358ea3c03e476ce80bfae1dc956852730cee2948", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/105/wire.bin", + "outcome": "unlinked", + "sha256": "ca94a63ef1a73229218199860a5391e858e766874ebc1b7362a06f65228d4aad", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/fields.zip", + "outcome": "unlinked", + "sha256": "7ad4b8d7e5de6b4a3aec1539062661df7551f74f66f0fe92285802dad3e99cd9", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/106/wire.bin", + "outcome": "unlinked", + "sha256": "6cc696471fe628586099d19f15b2202093ffebdcd08e6b0c27663473b0cc8a51", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/fields.zip", + "outcome": "unlinked", + "sha256": "c9ceee9e50861ee589b3e1c324d5dcb187cf4f79322fd12d208e450e86544cc1", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/107/wire.bin", + "outcome": "unlinked", + "sha256": "fff88c99b9d04ebf30440e66155fe4e750083295989bca635ab730e6498f64f2", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/fields.zip", + "outcome": "unlinked", + "sha256": "3043098224dc4fedf7ba737e0238b785fa136c02816ee1297a00ccd6524680bc", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/108/wire.bin", + "outcome": "unlinked", + "sha256": "db690f103de97e19d2de68beebe6d735415a8216a21f1c35559634944cd0802a", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/fields.zip", + "outcome": "unlinked", + "sha256": "93e4dd14ea507948ab329a08ae86282d4548422e59fa88d2644e439f1138db1d", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/109/wire.bin", + "outcome": "unlinked", + "sha256": "8fe7fd42d00fd96007941a853d1cacd6324b626e3b090cab4d86c7078ef8a713", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/fields.zip", + "outcome": "unlinked", + "sha256": "54f44df2fe839f283511d125a5000138070243552504df883b349455d63e46c3", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/110/wire.bin", + "outcome": "unlinked", + "sha256": "d0fc71906b63ed46038380f1ac09f211e48b4a44d4ce86dfd47210f9eaa2c259", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/fields.zip", + "outcome": "unlinked", + "sha256": "dcd360f9f6c68c557e610385745f1f036447848aadb27126a68b729d273d667e", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/native-diagnostics.txt", + "outcome": "unlinked", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "allocated_bytes": 0 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-OgbEYN/generation/111/wire.bin", + "outcome": "unlinked", + "sha256": "886d70ea42c70216ab285fde707b765c2fa81eeb1a10b4f4d5f10c371bb71bb4", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-iREMrq/generation/100/fields.zip", + "outcome": "unlinked", + "sha256": "d801973641c7ebd42b193c37b9c06053d0db59a4ab34dcfd7fcddf8a82d53600", + "allocated_bytes": 17027072 + }, + { + "path": "/workspace/Tuldok-p5b/build/qa/workspace-cpu-training/run-iREMrq/generation/100/wire.bin", + "outcome": "unlinked", + "sha256": "851969c2e6cfe8782078247271a13b950dcc26e48201a6e3af76bb03b844e4b6", + "allocated_bytes": 8491008 + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-4a9a129e8405405ab12b866edd766e48/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-48dbf653056a4d45ae53c6842e7d3aa3/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cea0a01e08c4488cbdba92f1662c7020/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-ecd6685cbd034a2e9c4c870b68ddd3a4/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-44272474e32544898feb39f651d997b9/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8a22050ad7fc496db53e67d091de062a/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-80ad7f8474ae4e9384483b9463206ab2/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-0cc3ef6b5f9e460f873c8eda1ba3d525/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-8d2985a122b644fa96f3bafd3146edf5/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-df53534ae6154ec8a4788f8378a8f004/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-d1700cfb1fb249e3b627a9cb1daadcec/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-3a0e69f8e71c4c7995d21c8157edc230/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-c3f7594d311843ea8487427556c84810/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-11fc2be1aada490f9b67c11c190728f6/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-e46bc5a79331428580dba01bb671d25b/output", + "outcome": "empty_directory_removed" + }, + { + "path": "/workspace/scratch/tuldok-cpu-qualification/qualified-catalog/workspace-runs/.external-cf94711d8a2047c08a64c0219f474017/output", + "outcome": "empty_directory_removed" + } + ], + "free_before": 7393628160, + "status": "completed", + "run_states": { + "4a9a129e8405405ab12b866edd766e48": "completed", + "48dbf653056a4d45ae53c6842e7d3aa3": "completed", + "28a3dd42f3e64fab89136da1d7e5d8a0": "failed", + "cea0a01e08c4488cbdba92f1662c7020": "completed", + "ab46bb3b6e2944a0a0794776da0352f7": "failed", + "ecd6685cbd034a2e9c4c870b68ddd3a4": "completed", + "44272474e32544898feb39f651d997b9": "completed", + "8a22050ad7fc496db53e67d091de062a": "completed", + "a8298e4e255a46d4beef0ec9f73e38d0": "cancelled", + "80ad7f8474ae4e9384483b9463206ab2": "completed", + "0cc3ef6b5f9e460f873c8eda1ba3d525": "completed", + "ece8abe0a4024d1483048d91b7980cf0": "failed", + "8d2985a122b644fa96f3bafd3146edf5": "completed", + "df53534ae6154ec8a4788f8378a8f004": "completed", + "d1700cfb1fb249e3b627a9cb1daadcec": "completed", + "3a0e69f8e71c4c7995d21c8157edc230": "completed", + "c3f7594d311843ea8487427556c84810": "completed", + "11fc2be1aada490f9b67c11c190728f6": "completed", + "e46bc5a79331428580dba01bb671d25b": "completed", + "cf94711d8a2047c08a64c0219f474017": "completed" + }, + "completed_publication_hashes_reverified": 16, + "excluded_staging_fingerprints": { + "28a3dd42f3e64fab89136da1d7e5d8a0": "3f2f0c49da4fa001ec3ea7b465163a7bb74a91c5c849b1742a21a493251b8389", + "ab46bb3b6e2944a0a0794776da0352f7": "18069a0fd369a4fd7cde298497b06ae4a79823b4180f28e0fec8f076e9c99804", + "a8298e4e255a46d4beef0ec9f73e38d0": "6434af70ec17e62655258739cb990e484192b1ea3f6bf490287f00eb6873d7d2", + "ece8abe0a4024d1483048d91b7980cf0": "cf5af400c795e5bebf4d21ef6ffeb37138de9248fe850eb69cd276b0e6b878ff" + }, + "free_after": 8301801472, + "measured_free_delta": 908173312 +} diff --git a/docs/plans/generic-scientific-projects/reports/p5c/installer-cleanup.json b/docs/plans/generic-scientific-projects/reports/p5c/installer-cleanup.json new file mode 100644 index 00000000..9202d920 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p5c/installer-cleanup.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "action": "remove_verified_installer_extraction", + "target": "/workspace/scratch/tuldok-science-probe/rust-1.92.0-x86_64-unknown-linux-gnu", + "allocated_bytes_observed": 1685254144, + "authorization": "Parent explicitly approved this exact extracted installer directory only.", + "archive_sha256": "d2ccef59dd9f7439f2c694948069f789a044dc1addcc0803613232af8f88ee0c", + "installed_and_extracted_rustc_sha256": "028fd60b0e0add5505c661cd3ccde91393d615c8729bd95d94ab1e91409b36a1", + "installed_and_extracted_cargo_sha256": "310fa46c7903259368c0df3dbb3f4f52759563f3c9ce53b500404b7741fb6860", + "reference_checks": { + "receipt_capture_qa_json": "No extraction path reference found in inspected build/capture receipts and QA root JSON.", + "installed_toolchain": "No symlinks outside install root; rustc/cargo exact hashes match extraction.", + "installation": "install.log records successful rustc/cargo/rust-std installation." + }, + "preserved": [ + "archive", + "archive checksum", + "verified-toolchain.sha256", + "install.log", + "toolchain", + "lab and Rheon sources", + "target binaries", + "build and capture receipts" + ], + "free_bytes_before": 6676639744, + "started_utc": "2026-10-13T20:02:31.113318+00:00", + "completed_utc": "2026-10-13T20:02:31.937642+00:00", + "target_absent": true, + "free_bytes_after": 8361893888, + "net_free_bytes_increase": 1685254144 +} diff --git a/pipeline.py b/pipeline.py index b8514426..19f94ed1 100644 --- a/pipeline.py +++ b/pipeline.py @@ -227,7 +227,7 @@ def preflight_external(self, graph_id, revision): if graph.get('recipe') != 'external_recipe' or type(revision) is not int or graph['revision'] != revision: raise WorkbenchError('External graph changed or is unavailable.', 'stale', 409) result, _captured = recipe_contract.prepare(self.dataset.projects, graph, execution_digest(graph), - getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases) + getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases, runs=getattr(self.dataset, 'runs', None)) if self.get(graph_id) != graph: raise WorkbenchError('Graph changed during preflight.', 'stale', 409) return result @@ -239,7 +239,7 @@ def capture_external(self, graph_id, revision, expected_digest, staging): if graph.get('recipe') != 'external_recipe' or type(revision) is not int or graph['revision'] != revision: raise WorkbenchError('External graph changed or is unavailable.', 'stale', 409) result, captured = recipe_contract.prepare(self.dataset.projects, graph, execution_digest(graph), - getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases) + getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases, runs=getattr(self.dataset, 'runs', None)) if expected_digest != result['material_sha256'] or self.get(graph_id) != graph: raise WorkbenchError('Reviewed execution material changed. Preflight again.', 'stale', 409) return dict(recipe_contract.capture(result, captured, staging), graph=graph) @@ -448,7 +448,7 @@ def validate_graph(self, graph): graph = decode_graph(graph) if graph.get('recipe') == 'external_recipe': result, _captured = recipe_contract.prepare(self.dataset.projects, graph, execution_digest(graph), - getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases) + getattr(self.dataset, 'external_runtime_profile', None), releases=self.dataset.releases, runs=getattr(self.dataset, 'runs', None)) return dict(result, errors=[], release=None) adapter = adapter_for_graph(graph) ports = ports_for_graph(graph) diff --git a/recipe_contract.py b/recipe_contract.py index a1610e1e..1bab7f5f 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -22,6 +22,8 @@ PORTS = {'input': (None, 'data'), 'execute': ('data', 'artifacts'), 'output': ('artifacts', None)} TRUST = 'Trusted local code; filesystem and network are not sandboxed. Runtime installation must remain immutable for the attempt.' CPU_PROFILE = 'captured_cpu_python_v1' +LARGE_OUTPUT_PROFILE = 'captured_cpu_large_outputs_v1' +INPUT_EVIDENCE = 'runtime-bootstrap/input-evidence.json' CPU_DESCRIPTOR = 'runtime-bootstrap/snapshot.json' CPU_BOOTSTRAP = 'runtime-bootstrap/recipe_process.py' HOST_NAMES = ('recipe_contract.py', 'projects.py', 'pipeline.py', 'recipe_process.py') @@ -127,11 +129,15 @@ def decode_definition(raw): if not isinstance(arg, str) or arg not in paths: raise WorkbenchError('Unknown source argument.') elif kind == 'native_policy': if not isinstance(arg,str) or arg not in value.get('native_sources',[]): raise WorkbenchError('Native policy argument must select a declared native source.') - elif kind != 'output_dir' or arg is not True: + elif kind not in ('output_dir', 'input_evidence') or arg is not True: raise WorkbenchError('Unsupported argument token.') + closed(value['resources'], ('wall_seconds', 'max_output_bytes'), ('profile',)) + large = 'profile' in value['resources'] + if large and (value['resources']['profile'] != LARGE_OUTPUT_PROFILE or value['runtime'] != CPU_PROFILE): + raise WorkbenchError('Large outputs require the declared captured CPU resource profile.') outputs = value['outputs'] - if not isinstance(outputs, list) or not 1 <= len(outputs) <= 32: - raise WorkbenchError('Declare 1–32 output files.') + if not isinstance(outputs, list) or not 1 <= len(outputs) <= (96 if large else 32): + raise WorkbenchError('Output file count exceeds the resource profile bound.') seen = set() for output in outputs: closed(output, ('path', 'role', 'max_bytes', 'required')) @@ -140,12 +146,11 @@ def decode_definition(raw): raise WorkbenchError('Output must use an unreserved flat filename.') if path in seen or type(output['required']) is not bool: raise WorkbenchError('Invalid output contract.') seen.add(path) - bounded_int(output['max_bytes'], 1, MAX_CAPTURE_BYTES, 'Output bytes') + bounded_int(output['max_bytes'], 1, 512 * 1024 * 1024 if large else MAX_CAPTURE_BYTES, 'Output bytes') if value['environment'] != {}: raise WorkbenchError('Initial runtime admits no project environment overrides.', 'unsupported', 400) - closed(value['resources'], ('wall_seconds', 'max_output_bytes')) - bounded_int(value['resources']['wall_seconds'], 1, 600, 'Wall seconds') - bounded_int(value['resources']['max_output_bytes'], 1, MAX_CAPTURE_BYTES, 'Output bytes') + bounded_int(value['resources']['wall_seconds'], 1, 2400 if large else 600, 'Wall seconds') + bounded_int(value['resources']['max_output_bytes'], 1, 4 * 1024**3 if large else MAX_CAPTURE_BYTES, 'Output bytes') return value @@ -425,7 +430,7 @@ def host_identity(): return current -def prepare(projects, graph, execution_sha256, profile=None, *, releases=None): +def prepare(projects, graph, execution_sha256, profile=None, *, releases=None, runs=None): config, values = graph_parameters(graph) inspected = inspect_recipe(projects, config['project_id'], config['recipe_id'], config['expected_sha256']) definition = inspected['definition']; values = validate_parameters(definition, values) @@ -435,7 +440,14 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None): captured = {'source/' + path: raw for path, raw in sources.items()} inputs = {} for name, binding in config['inputs'].items(): - if isinstance(binding, dict) and binding.get('kind') == 'frozen_release': + if isinstance(binding, dict) and binding.get('kind') == 'run_artifact': + closed(binding, ('kind', 'run_id', 'artifact_name', 'expected_sha256')) + if runs is None: + raise WorkbenchError('Run artifact input owner is unavailable.', 'unavailable', 409) + raw, source = runs.capture_artifact(binding['run_id'], binding['artifact_name'], binding['expected_sha256'], + max_bytes=definition['inputs'][name]['max_bytes']) + receipt = {'kind': 'run_artifact', 'source': source} + elif isinstance(binding, dict) and binding.get('kind') == 'frozen_release': closed(binding, ('kind', 'release_id', 'expected_sha256')) if releases is None: raise WorkbenchError('Frozen release input owner is unavailable.', 'unavailable', 409) @@ -444,6 +456,8 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None): closed(binding, ('reference_id', 'expected_sha256')) raw, receipt = projects.read_data_reference(config['project_id'], binding['reference_id'], binding['expected_sha256'], definition['inputs'][name]['max_bytes']) captured['input/' + name] = raw; inputs[name] = receipt + if {'input_evidence': True} in definition['arguments']: + captured[INPUT_EVIDENCE] = encode(inputs).encode() capture_limit = CPU_CAPTURE_BYTES if definition['runtime'] == CPU_PROFILE else MAX_CAPTURE_BYTES input_limit = CPU_INPUT_BYTES if definition['runtime'] == CPU_PROFILE else MAX_CAPTURE_BYTES if sum(len(raw) for path,raw in captured.items() if path.startswith('input/')) > input_limit: @@ -510,7 +524,23 @@ def receipt(value): if not isinstance(material['inputs'], dict) or set(material['inputs']) != set(definition['inputs']): raise WorkbenchError('Material input inventory differs.') expected = {'source/'+item['path']:item['sha256'] for item in definition['sources']} for name, item in material['inputs'].items(): - if isinstance(item, dict) and item.get('kind') == 'frozen_release': + if isinstance(item, dict) and item.get('kind') == 'run_artifact': + import recipe_process + closed(item, ('kind', 'source')) + source = item['source'] + capture = material['captures'].get('input/' + name) if isinstance(material['captures'], dict) else None + if not isinstance(capture, dict) or not isinstance(source, dict): + raise WorkbenchError('Run input capture or receipt is missing.') + try: + recipe_process.validate_artifact_source(source, capture.get('sha256'), capture.get('bytes')) + except recipe_process.ProcessError as error: + raise WorkbenchError('Invalid Run input custody: ' + str(error)) from None + if source['catalog_id'] != material['catalog_id']: + raise WorkbenchError('Run input belongs to another catalog.') + bounded_int(source['artifact_bytes'], 0, definition['inputs'][name]['max_bytes'], 'Run input bytes') + expected['input/' + name] = source['measured_sha256'] + continue + elif isinstance(item, dict) and item.get('kind') == 'frozen_release': closed(item, ('kind','catalog_id','release_id','measured_sha256','bytes')) sha(item['release_id']); sha(item['measured_sha256']) if item['catalog_id'] != material['catalog_id'] or item['release_id'] != item['measured_sha256']: @@ -522,6 +552,8 @@ def receipt(value): raise WorkbenchError('Input receipt belongs to another project snapshot.') expected['input/'+name] = item['measured_sha256'] generated = validate_native_policies(material,definition) + if {'input_evidence': True} in definition['arguments']: + generated[INPUT_EVIDENCE] = encode(material['inputs']).encode() if definition['runtime'] == CPU_PROFILE: import recipe_process try: @@ -598,6 +630,7 @@ def spec_for_material(material, staging): elif kind == 'input': arg = str(staging/'input'/value) elif kind == 'source': arg = str(staging/'source'/value) elif kind == 'native_policy': arg = str(staging/native_policy_path(value)) + elif kind == 'input_evidence': arg = str(staging/INPUT_EVIDENCE) else: arg = str(output) argv.append(arg) spec = {'schema_version':1,'argv':argv,'env':dict(material['environment']),'cwd':str(staging), 'stdin_base64':base64.b64encode(b'').decode(), @@ -607,6 +640,7 @@ def spec_for_material(material, staging): 'trusted_install_owner_uid':runtime['trusted_install_owner_uid']} if 'native_policies' in material or 'cpu_snapshot' in material: spec['native_loader_policy'] = NATIVE_POLICY if 'cpu_snapshot' in material: spec['cpu_snapshot'] = material['cpu_snapshot'] + if 'profile' in definition['resources']: spec['resource_profile'] = definition['resources']['profile'] return spec @@ -640,28 +674,60 @@ def capture(preflight, captured, staging): def verify_outputs(output_dir, outputs, max_output_bytes): - root = Path(output_dir) - if root.is_symlink() or not root.is_dir(): raise WorkbenchError('Output root is not a regular directory.') - allowed = {entry['path']:entry for entry in outputs}; found = {}; total = 0 - for directory, dirs, files in os.walk(root, followlinks=False): - if any((Path(directory)/name).is_symlink() for name in dirs): raise WorkbenchError('Output directory symlinks are forbidden.') - for name in files: - path = Path(directory)/name; relative = path.relative_to(root).as_posix() - if relative not in allowed or len(found) >= MAX_FILES: raise WorkbenchError('Undeclared output file.') - fd = os.open(path, os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK) - try: - info = os.fstat(fd); rule = allowed[relative] - if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > rule['max_bytes']: raise WorkbenchError('Output must be a bounded unlinked regular file.') - chunks = []; remaining = rule['max_bytes']+1 - while remaining: - raw = os.read(fd,min(65536,remaining)) - if not raw: break - chunks.append(raw); remaining -= len(raw) - raw = b''.join(chunks); after = os.fstat(fd) - if len(raw) != info.st_size or (info.st_size,info.st_mtime_ns,info.st_ctime_ns) != (after.st_size,after.st_mtime_ns,after.st_ctime_ns): raise WorkbenchError('Output changed during verification.') - total += len(raw) - if total > max_output_bytes: raise WorkbenchError('Output total exceeds declared limit.') - found[relative] = {'path':relative,'role':rule['role'],'bytes':len(raw),'sha256':file_digest(raw)} - finally: os.close(fd) - if any(item['required'] and item['path'] not in found for item in outputs): raise WorkbenchError('Required output is missing.') - return list(found.values()) + """Hash declared outputs with bounded memory and anchored file descriptors.""" + root = Path(output_dir).absolute() + root_fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY) + try: + for part in root.parts[1:]: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_fd) + os.close(root_fd) + root_fd = child + root_info = os.fstat(root_fd) + allowed = {entry['path']: entry for entry in outputs} + found = {} + total = 0 + identity = lambda info: (info.st_dev, info.st_ino, info.st_mode, info.st_uid, + info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + for directory, dirs, files, directory_fd in os.fwalk('.', dir_fd=root_fd, follow_symlinks=False): + if any(stat.S_ISLNK(os.stat(name, dir_fd=directory_fd, follow_symlinks=False).st_mode) for name in dirs): + raise WorkbenchError('Output directory symlinks are forbidden.') + for name in files: + relative = (Path(directory) / name).as_posix() + if relative not in allowed or len(found) >= MAX_FILES: + raise WorkbenchError('Undeclared output file.') + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd) + try: + info = os.fstat(fd) + rule = allowed[relative] + if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 + or info.st_uid != os.getuid() or info.st_size > rule['max_bytes']): + raise WorkbenchError('Output must be a bounded unlinked regular file.') + measured = 0 + checksum = hashlib.sha256() + while True: + raw = os.read(fd, min(1024 * 1024, rule['max_bytes'] + 1 - measured)) + if not raw: + break + measured += len(raw) + if measured > rule['max_bytes'] or total + measured > max_output_bytes: + raise WorkbenchError('Output total exceeds declared limit.') + checksum.update(raw) + after = os.fstat(fd) + current = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + if measured != info.st_size or identity(info) != identity(after) or identity(info) != identity(current): + raise WorkbenchError('Output changed during verification.') + total += measured + found[relative] = {'path': relative, 'role': rule['role'], + 'bytes': measured, 'sha256': checksum.hexdigest()} + finally: + os.close(fd) + current_root = os.stat(root, follow_symlinks=False) + if (root_info.st_dev, root_info.st_ino) != (current_root.st_dev, current_root.st_ino): + raise WorkbenchError('Output root changed during verification.') + if any(item['required'] and item['path'] not in found for item in outputs): + raise WorkbenchError('Required output is missing.') + return list(found.values()) + except OSError as error: + raise WorkbenchError('Output cannot be safely verified.', 'unavailable', 409) from error + finally: + os.close(root_fd) diff --git a/recipe_process.py b/recipe_process.py index ab9ca646..0dbc0e49 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -24,6 +24,10 @@ CLEANUP_SECONDS = 10 STREAM_LIMIT = 1024 * 1024 OUTPUT_LIMIT = 64 * 1024 * 1024 +CAPTURE_FILE_LIMIT = 64 * 1024 * 1024 +LARGE_OUTPUT_PROFILE = 'captured_cpu_large_outputs_v1' +LARGE_OUTPUT_LIMIT = 4 * 1024 * 1024 * 1024 +LARGE_OUTPUT_FILE_LIMIT = 512 * 1024 * 1024 SPEC_LIMIT = 4 * 1024 * 1024 SOURCE = Path(__file__).read_bytes() HASH = re.compile(r'^[a-f0-9]{64}$') @@ -298,7 +302,7 @@ def verify_cpu_snapshot(descriptor, *, checkpoint=None): def validate_spec(value): keys = {'schema_version','attempt_id','request_sha256','argv','env','cwd','stdin_base64', 'wall_seconds','executable_sha256','max_output_bytes','output_dir','captured_files','runtime_files','trusted_install_owner_uid'} - if type(value) is not dict or set(value) - {'native_loader_policy', 'cpu_snapshot'} != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: + if type(value) is not dict or set(value) - {'native_loader_policy', 'cpu_snapshot', 'resource_profile'} != keys or type(value['schema_version']) is not int or value['schema_version'] != 1: raise ProcessError('Unsupported custody spec.') if not isinstance(value['attempt_id'], str) or not ID.fullmatch(value['attempt_id']): raise ProcessError('Invalid attempt identity.') @@ -314,10 +318,6 @@ def validate_spec(value): raise ProcessError('Invalid explicit environment.') if not absolute(value['cwd']) or not absolute(value['output_dir']): raise ProcessError('Captured working/output directories must be absolute.') - if type(value['wall_seconds']) is not int or not 1 <= value['wall_seconds'] <= 600: - raise ProcessError('Wall budget must be 1 through 600 seconds.') - if type(value['max_output_bytes']) is not int or not 1 <= value['max_output_bytes'] <= OUTPUT_LIMIT: - raise ProcessError('Invalid output byte budget.') try: if type(value['stdin_base64']) is not str or len(value['stdin_base64']) > 87384: raise ValueError() @@ -340,7 +340,7 @@ def validate_spec(value): if item['path'] in seen or type(item['sha256']) is not str or not HASH.fullmatch(item['sha256']): raise ProcessError('Duplicate capture or invalid hash.') seen.add(item['path']) - if type(item['bytes']) is not int or not 0 <= item['bytes'] <= OUTPUT_LIMIT: + if type(item['bytes']) is not int or not 0 <= item['bytes'] <= CAPTURE_FILE_LIMIT: raise ProcessError('Invalid captured file size.') if 'cpu_snapshot' in value: descriptor = validate_cpu_snapshot_descriptor(value['cpu_snapshot']) @@ -374,6 +374,16 @@ def validate_spec(value): raise ProcessError('CPU input aggregate exceeds its byte bound.') if sum(item['bytes'] for item in files) > capture_limit or sum(item['bytes'] for item in runtime) > 512 * 1024 * 1024: raise ProcessError('Runtime/source closure exceeds its byte bound.') + wall_limit, output_limit = 600, OUTPUT_LIMIT + if 'resource_profile' in value: + if value['resource_profile'] != LARGE_OUTPUT_PROFILE or 'cpu_snapshot' not in value: + raise ProcessError('Large outputs require the explicit qualified CPU resource profile.') + # CPU descriptor, launch vector and capture layout have all passed above. + wall_limit, output_limit = 2400, LARGE_OUTPUT_LIMIT + if type(value['wall_seconds']) is not int or not 1 <= value['wall_seconds'] <= wall_limit: + raise ProcessError('Wall budget exceeds its admitted profile.') + if type(value['max_output_bytes']) is not int or not 1 <= value['max_output_bytes'] <= output_limit: + raise ProcessError('Invalid output byte budget.') if len(encoded(value)) > SPEC_LIMIT: raise ProcessError('Custody specification exceeds its bound.') return decode(encoded(value)) @@ -621,6 +631,62 @@ def validate_receipt(raw, expected_identity): return value +def validate_artifact_source(source, expected_sha256, expected_bytes): + """Check retained Run artifact custody declarations, not historical authority.""" + if (type(expected_sha256) is not str or not HASH.fullmatch(expected_sha256) + or type(expected_bytes) is not int or expected_bytes < 0): + raise ProcessError('Invalid expected artifact identity.') + def exact(value, keys, label): + if type(value) is not dict or set(value) != set(keys): + raise ProcessError(label + ' fields differ from schema 1.') + exact(source, ('schema_version', 'catalog_id', 'run_id', 'attempt_id', + 'request_sha256', 'material_sha256', 'artifact_name', + 'artifact_role', 'artifact_bytes', 'measured_sha256', + 'custody_receipt_sha256', 'custody_identity', + 'custody_receipt_base64'), 'Run source') + if type(source['schema_version']) is not int or source['schema_version'] != 1: + raise ProcessError('Unsupported Run source version.') + for key in ('catalog_id', 'run_id', 'attempt_id'): + if type(source[key]) is not str or not re.fullmatch('[a-f0-9]{32}', source[key]): + raise ProcessError('Invalid Run source identity.') + for key in ('request_sha256', 'material_sha256', 'measured_sha256', 'custody_receipt_sha256'): + if type(source[key]) is not str or not HASH.fullmatch(source[key]): + raise ProcessError('Invalid Run source hash.') + if (type(source['artifact_name']) is not str + or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,119}', source['artifact_name']) + or source['artifact_name'] in ('artifact-manifest.json', 'custody-receipt.json') + or type(source['artifact_role']) is not str or not re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$').fullmatch(source['artifact_role'])): + raise ProcessError('Invalid Run artifact declaration.') + if type(source['artifact_bytes']) is not int or source['artifact_bytes'] != expected_bytes: + raise ProcessError('Run artifact size differs from captured bytes.') + identity = source['custody_identity'] + exact(identity, ('attempt_id', 'request_sha256', 'spec_sha256', 'helper'), 'Custody identity') + if (source['run_id'] != source['attempt_id'] or source['attempt_id'] != identity['attempt_id'] + or source['request_sha256'] != identity['request_sha256'] + or type(identity['spec_sha256']) is not str or not HASH.fullmatch(identity['spec_sha256'])): + raise ProcessError('Run source and custody identities differ.') + encoded = source['custody_receipt_base64'] + if type(encoded) is not str or not 1 <= len(encoded) <= 21848: + raise ProcessError('Invalid retained custody receipt encoding.') + try: + receipt_raw = base64.b64decode(encoded, validate=True) + except ValueError as error: + raise ProcessError('Invalid retained custody receipt encoding.') from error + if (not 0 < len(receipt_raw) <= 16384 + or base64.b64encode(receipt_raw).decode('ascii') != encoded + or hashlib.sha256(receipt_raw).hexdigest() != source['custody_receipt_sha256']): + raise ProcessError('Retained custody receipt bytes differ.') + try: + receipt = validate_receipt(receipt_raw, identity) + except ProcessError as error: + raise ProcessError('Invalid retained custody evidence.') from error + if receipt['outcome'] != 'completed' or not receipt['drained']: + raise ProcessError('Run source requires completed and drained custody.') + if source['measured_sha256'] != expected_sha256: + raise ProcessError('Captured bytes differ from their source receipt.', 'conflict') + return dict(source) + + def _children(): # /proc/.../children is not exposed on every supported host. PPID is the # admitted canonical observation; ECHILD, never this scan, proves drainage. @@ -651,11 +717,13 @@ def _signal_children(number): def _outputs(spec): root=Path(spec['output_dir']); total=count=0 + large = spec.get('resource_profile') == LARGE_OUTPUT_PROFILE + count_limit, file_limit = (96, LARGE_OUTPUT_FILE_LIMIT) if large else (128, OUTPUT_LIMIT) if root.is_symlink() or not root.is_dir(): raise ProcessError('Output directory changed.') for path in root.iterdir(): info=path.lstat();count+=1;total+=info.st_size - if count>128 or not stat.S_ISREG(info.st_mode) or total>spec['max_output_bytes']: + if count>count_limit or not stat.S_ISREG(info.st_mode) or info.st_size>file_limit or total>spec['max_output_bytes']: raise ProcessError('Output inventory exceeded its admitted bound.') diff --git a/runs.py b/runs.py index b82203d7..3603cffd 100644 --- a/runs.py +++ b/runs.py @@ -2,6 +2,9 @@ from access import authorized import base64 import copy +from contextlib import contextmanager +import io +import tempfile import hashlib import json import math @@ -33,6 +36,7 @@ TEST_FILES = {'test-metrics.json', 'predictions-test.json'} REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$') MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 +_DOWNLOAD_PERMIT = threading.BoundedSemaphore(1) def process_identity(pid): @@ -249,7 +253,9 @@ def list(self): public = self._public(run) if run['state'] == 'completed': try: - self._verified_artifacts(run) + status = self._inspect_published(run) + if status: + public['artifact_status'] = status except WorkbenchError as error: public['metrics'] = None public.pop('validation_summary', None) @@ -261,10 +267,12 @@ def list(self): def get(self, run_id): with self.lock: run = self._load(run_id) + public = self._public(run) if run['state'] == 'completed': - # A retained completed envelope never certifies missing or altered files. - self._verified_artifacts(run) - return self._public(run) + status = self._inspect_published(run) + if status: + public['artifact_status'] = status + return public def _request(self, body, required): closed(body, required) @@ -334,6 +342,17 @@ def submit_external(self, body): if receipt: self.db.on_commit(lambda: shutil.rmtree(staging, ignore_errors=True)) else: + # Captured/validated resource policy owns this bound. Check + # after both reconciliation opportunities, before consuming + # approval or persisting admission; failure uses normal cleanup. + if spec.get('resource_profile') == 'captured_cpu_large_outputs_v1': + required_bytes = 2 * spec['max_output_bytes'] + 1024**3 + try: + available_bytes = shutil.disk_usage(self.path).free + except OSError as error: + raise WorkbenchError('Execution storage capacity is unavailable.', 'unavailable', 409) from error + if available_bytes < required_bytes: + raise WorkbenchError('Insufficient storage for declared outputs, publication copy and reserve.', 'unavailable', 409) if self.pipelines._get(graph['id']) != graph: raise WorkbenchError('Graph changed during capture.', 'stale', 409) if sum(self._load(row[0])['state'] in ACTIVE for row in self.db.execute('SELECT id FROM workspace_runs')) >= 8: @@ -503,9 +522,10 @@ def _external_result_summary(self, run): try: if len(rules) != 1 or rules[0]['max_bytes'] > 65536: raise ValueError('Invalid result summary declaration.') - inventory = self._verified_artifacts(run) + inventory = (self._external_inventory(run) if self._large_outputs(run) + else self._verify_external_artifacts(run)) item = next(item for item in inventory if item['name'] == rules[0]['path'] and item['role'] == 'result_summary') - raw, _media = self.artifact(run['id'], item['name']) + raw = self._external_bytes(run, item) if len(raw) > 65536: raise ValueError('Result summary is too large.') value = self._progress_json(raw) @@ -527,6 +547,13 @@ def _external_result_summary(self, run): raise ValueError('Invalid scope.') if 'checkpoint' in value and not any(item['name'] == value['checkpoint'] and item['role'] == 'checkpoint' for item in inventory): raise ValueError('Checkpoint is not a verified declared output.') + if 'checkpoint' in value: + checkpoint = next(entry for entry in inventory if entry['name'] == value['checkpoint']) + directory = self._diagnostic_directory(self.path / run['id']) + try: + self._copy_verified(directory, checkpoint['name'], checkpoint) + finally: + os.close(directory) return {'status':'hash_verified','artifact_name':item['name'], 'sha256':item['sha256'], 'summary':value} except (OSError, ValueError, TypeError, KeyError, StopIteration, OverflowError, RecursionError): return {'status':'unavailable'} @@ -592,19 +619,25 @@ def _publish_external(self, run, receipt): destination.mkdir(mode=0o700) artifacts = [] total = 0 - for item in checked: - raw = self._read_regular(source / item['path'], item['bytes']) - if len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != item['sha256']: - raise WorkbenchError('Recipe output changed after validation.', 'stale', 409) - total += len(raw) - if total > run['spec']['max_output_bytes']: - raise WorkbenchError('Recipe outputs exceed their admitted bound.', 'invalid', 409) - with (destination / item['path']).open('xb') as output: - output.write(raw) - output.flush() - os.fsync(output.fileno()) - artifacts.append({'name': item['path'], 'role': item['role'], 'bytes': len(raw), - 'sha256': item['sha256'], 'media_type': item.get('media_type', 'application/octet-stream')}) + source_descriptor = self._diagnostic_directory(source) + try: + if shutil.disk_usage(self.path).free < sum(item['bytes'] for item in checked) + 512*1024*1024: + raise WorkbenchError('Insufficient space for artifact publication.', 'unavailable', 409) + for item in checked: + total += item['bytes'] + if total > run['spec']['max_output_bytes']: + raise WorkbenchError('Recipe outputs exceed their admitted bound.', 'invalid', 409) + try: + with (destination / item['path']).open('xb') as output: + self._copy_verified(source_descriptor, item['path'], item, output) + output.flush() + os.fsync(output.fileno()) + except (OSError, ValueError) as error: + raise WorkbenchError('Recipe output changed or could not be published.', 'unavailable', 409) from error + artifacts.append({'name':item['path'],'role':item['role'],'bytes':item['bytes'], + 'sha256':item['sha256'],'media_type':item.get('media_type','application/octet-stream')}) + finally: + os.close(source_descriptor) receipt_raw = encode(receipt).encode() manifest = {'schema_version': 1, 'run_id': run['id'], 'material_sha256': run['material_sha256'], 'request_sha256': run['request_sha256'], 'custody_receipt_sha256': hashlib.sha256(receipt_raw).hexdigest(), @@ -707,11 +740,15 @@ def _execute_external(self, run_id): # to stop and drain; missing evidence remains unknown on reopen. handle.close_control() - def _verify_external_artifacts(self, run): + def _external_inventory(self, run): import recipe_process import recipe_contract directory = self.path / run['id'] + descriptor = None try: + descriptor = self._diagnostic_directory(directory) + if run['state'] != 'completed': + raise ValueError('Only completed Runs publish artifacts.') if directory.is_symlink() or not directory.is_dir(): raise ValueError('Invalid artifact directory.') definition = recipe_contract.decode_definition(encode(run['material']['recipe']).encode()) @@ -732,25 +769,175 @@ def _verify_external_artifacts(self, run): if (any(rule['required'] and name not in seen for name, rule in allowed.items()) or total > definition['resources']['max_output_bytes']): raise ValueError('Incomplete or excessive artifact inventory.') - receipt_raw = self._read_regular(directory / 'custody-receipt.json', 16384) + receipt_raw, _, _ = self._diagnostic_read(descriptor, 'custody-receipt.json', 16384) if hashlib.sha256(receipt_raw).hexdigest() != run['custody_receipt_sha256']: raise ValueError('Custody receipt changed.') - receipt = recipe_process.load_receipt(directory / 'custody-receipt.json', run['custody_identity']) + receipt = recipe_process.validate_receipt(receipt_raw, run['custody_identity']) if not receipt['drained'] or receipt['outcome'] != 'completed': raise ValueError('Custody did not complete.') - manifest = json.loads(self._read_regular(directory / 'artifact-manifest.json', 65536)) + manifest_raw, _, _ = self._diagnostic_read(descriptor, 'artifact-manifest.json', 65536) + manifest = self._progress_json(manifest_raw) expected = {'schema_version': 1, 'run_id': run['id'], 'material_sha256': run['material_sha256'], 'request_sha256': run['request_sha256'], 'custody_receipt_sha256': run['custody_receipt_sha256'], 'artifacts': run['artifacts']} if manifest != expected: raise ValueError('Artifact manifest changed.') + if set(os.listdir(descriptor)) != {item['name'] for item in run['artifacts']} | {'artifact-manifest.json','custody-receipt.json'}: + raise ValueError('Published directory inventory differs.') for item in run['artifacts']: - raw = self._read_regular(directory / item['name'], item['bytes']) - if len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != item['sha256']: - raise ValueError('Artifact changed.') + info = os.stat(item['name'], dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size != item['bytes']: + raise ValueError('Artifact inventory changed.') return run['artifacts'] - except (OSError, ValueError, KeyError, TypeError) as error: + except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: + raise WorkbenchError('Retained external result cannot be verified.', 'unavailable', 409) from error + finally: + if descriptor is not None: + os.close(descriptor) + + def _verify_external_artifacts(self, run): + inventory = self._external_inventory(run) + directory = None + try: + directory = self._diagnostic_directory(self.path / run['id']) + for item in inventory: + self._copy_verified(directory, item['name'], item) + return inventory + except (OSError, ValueError) as error: raise WorkbenchError('Retained external result cannot be verified.', 'unavailable', 409) from error + finally: + if directory is not None: + os.close(directory) + + @staticmethod + def _large_outputs(run): + return (run.get('execution_kind') == 'external_recipe' and + run['material']['recipe']['resources'].get('profile') == 'captured_cpu_large_outputs_v1') + + def _inspect_published(self, run): + if self._external(run) and self._large_outputs(run): + self._external_inventory(run) + return {'status':'receipt_verified', 'payloads':'not_checked'} + self._verified_artifacts(run) + return None + + @staticmethod + def _copy_verified(directory, name, item, output=None): + """Hash one exact regular payload, optionally copying with bounded memory.""" + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 + or before.st_uid != os.getuid() or before.st_size != item['bytes']): + raise ValueError('Artifact is not the declared regular file.') + total = 0 + measured = hashlib.sha256() + while True: + raw = os.read(fd, min(65536, item['bytes'] - total + 1)) + if not raw: + break + total += len(raw) + if total > item['bytes']: + raise ValueError('Artifact grew during verification.') + measured.update(raw) + if output is not None: + if output.write(raw) != len(raw): + raise OSError('Artifact copy made incomplete progress.') + after = os.fstat(fd) + current = os.stat(name, dir_fd=directory, follow_symlinks=False) + if (total != item['bytes'] or measured.hexdigest() != item['sha256'] + or (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns) != + (after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns) + or (before.st_dev,before.st_ino) != (current.st_dev,current.st_ino)): + raise ValueError('Artifact changed during verification.') + finally: + os.close(fd) + + def _selected_external(self, run, name, *, max_bytes=None): + inventory = self._external_inventory(run) + item = next((item for item in inventory if item['name'] == name), None) + if item is None: + raise WorkbenchError('Unknown external artifact.', 'unavailable', 404) + if max_bytes is not None and item['bytes'] > max_bytes: + raise WorkbenchError('Artifact exceeds the declared input byte bound.', 'invalid', 409) + if not self._large_outputs(run): + self._verify_external_artifacts(run) + return item + + def _external_bytes(self, run, item): + directory = None + try: + directory = self._diagnostic_directory(self.path / run['id']) + output = io.BytesIO() + self._copy_verified(directory, item['name'], item, output) + return output.getvalue() + except (OSError, ValueError) as error: + raise WorkbenchError('Selected artifact cannot be verified.', 'unavailable', 409) from error + finally: + if directory is not None: + os.close(directory) + + @authorized('inspect', 'runs.input_candidates') + def input_candidates(self): + with self.lock: + rows = self.db.execute('SELECT id FROM workspace_runs ORDER BY id').fetchall() + completed = [] + for row in rows: + run = self._load(row[0]) + if self._external(run) and run['state'] == 'completed': + completed.append(run) + completed = sorted(completed, key=lambda run:(run['created_at'],run['id']), reverse=True)[:100] + result = [] + for run in completed: + entry = {'run_id':run['id'], 'state':'completed', 'status':'receipt_verified', 'reason':None, 'artifacts':[]} + try: + inventory = self._external_inventory(run) + entry['artifacts'] = [{'artifact_name':item['name'],'role':item['role'],'bytes':item['bytes'], + 'sha256':item['sha256'],'status':'receipt_verified'} + for item in inventory if item['bytes'] <= 40*1024*1024] + except WorkbenchError: + entry.update(status='unavailable', reason='Completed artifact inventory cannot be verified.') + result.append(entry) + return {'runs':result} + + @authorized('inspect', 'runs.open_artifact') + def open_artifact(self, run_id, name): + return self._open_artifact(run_id, name) + + @contextmanager + def _open_artifact(self, run_id, name): + # Recheck at context entry: callers cannot retain an unentered context + # and use it after their credential has been revoked or unbound. + self.access.require('inspect', 'runs.open_artifact', run_id) + with self.lock: + run = self._load(run_id) + if not self._external(run): + raw, media_type = self.artifact(run_id, name) + with io.BytesIO(raw) as stream: + yield stream, {'bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest(),'media_type':media_type} + return + item = self._selected_external(run, name) + if not _DOWNLOAD_PERMIT.acquire(blocking=False): + raise WorkbenchError('Another artifact download is in progress.', 'unavailable', 503) + directory = None + try: + if shutil.disk_usage(self.path).free < item['bytes'] + 512*1024*1024: + raise WorkbenchError('Insufficient free space for a verified download.', 'unavailable', 409) + directory = self._diagnostic_directory(self.path / run['id']) + with tempfile.TemporaryFile(dir=self.path, mode='w+b') as stream: + try: + self._copy_verified(directory, name, item, stream) + stream.flush() + stream.seek(0) + except (OSError, ValueError) as error: + raise WorkbenchError('Selected artifact cannot be verified.', 'unavailable', 409) from error + yield stream, {key:item[key] for key in ('bytes','sha256','media_type')} + except OSError as error: + raise WorkbenchError('Verified download storage is unavailable.', 'unavailable', 409) from error + finally: + if directory is not None: + os.close(directory) + _DOWNLOAD_PERMIT.release() @authorized('execute', 'runs.submit', owner_only=True) def submit(self, body): @@ -1364,7 +1551,7 @@ def _verified_artifacts(self, run): return artifacts @authorized('inspect', 'runs.capture_artifact') - def capture_artifact(self, run_id, name, expected_sha256): + def capture_artifact(self, run_id, name, expected_sha256, *, max_bytes=MAX_ARTIFACT_BYTES): """Capture a completed external artifact with server-derived custody evidence.""" import recipe_process if type(expected_sha256) is not str or not re.fullmatch('[a-f0-9]{64}', expected_sha256): @@ -1373,12 +1560,12 @@ def capture_artifact(self, run_id, name, expected_sha256): run = self._load(run_id) if not self._external(run): raise WorkbenchError('Scientific import requires a completed external Run.', 'unsupported', 400) - item = next((item for item in self._verified_artifacts(run) if item['name'] == name), None) - if item is None: - raise WorkbenchError('Unknown external artifact.', 'unavailable', 404) + if type(max_bytes) is not int or not 0 < max_bytes <= MAX_ARTIFACT_BYTES: + raise WorkbenchError('Invalid artifact input byte bound.') + item = self._selected_external(run, name, max_bytes=max_bytes) if item['sha256'] != expected_sha256: raise WorkbenchError('Artifact differs from the expected SHA256.', 'conflict', 409) - raw = self._read_regular(self.path / run_id / name, item['bytes']) + raw = self._external_bytes(run, item) if len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != expected_sha256: raise WorkbenchError('Artifact changed during capture.', 'conflict', 409) receipt_raw = self._read_regular(self.path / run_id / 'custody-receipt.json', 16384) @@ -1405,10 +1592,8 @@ def artifact(self, run_id, name): with self.lock: run = self._load(run_id) if self._external(run): - item = next((item for item in self._verified_artifacts(run) if item['name'] == name), None) - if item is None: - raise WorkbenchError('Unknown external artifact.', 'unavailable', 404) - raw = self._read_regular(self.path / run_id / name, item['bytes']) + item = self._selected_external(run, name, max_bytes=MAX_ARTIFACT_BYTES) + raw = self._external_bytes(run, item) if len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != item['sha256']: raise WorkbenchError('Artifact changed during read.', 'unavailable', 409) return raw, item['media_type'] diff --git a/scientific_fields.py b/scientific_fields.py index 73f8c342..6174bb89 100644 --- a/scientific_fields.py +++ b/scientific_fields.py @@ -292,49 +292,10 @@ def validate_source(method, source, raw): text(source['path'], 512, 'Project source path') elif method == 'run_artifact_import': import recipe_process - exact(source, ('schema_version', 'catalog_id', 'run_id', 'attempt_id', - 'request_sha256', 'material_sha256', 'artifact_name', - 'artifact_role', 'artifact_bytes', 'measured_sha256', - 'custody_receipt_sha256', 'custody_identity', - 'custody_receipt_base64'), 'Run source') - if type(source['schema_version']) is not int or source['schema_version'] != 1: - invalid('Unsupported Run source version.') - for key in ('catalog_id', 'run_id', 'attempt_id'): - if type(source[key]) is not str or not re.fullmatch('[a-f0-9]{32}', source[key]): - invalid('Invalid Run source identity.') - for key in ('request_sha256', 'material_sha256', 'measured_sha256', 'custody_receipt_sha256'): - if type(source[key]) is not str or not HASH.fullmatch(source[key]): - invalid('Invalid Run source hash.') - if (type(source['artifact_name']) is not str - or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,119}', source['artifact_name']) - or source['artifact_name'] in ('artifact-manifest.json', 'custody-receipt.json') - or type(source['artifact_role']) is not str or not TOKEN.fullmatch(source['artifact_role'])): - invalid('Invalid Run artifact declaration.') - if type(source['artifact_bytes']) is not int or source['artifact_bytes'] != len(raw): - invalid('Run artifact size differs from captured bytes.') - identity = source['custody_identity'] - exact(identity, ('attempt_id', 'request_sha256', 'spec_sha256', 'helper'), 'Custody identity') - if (source['run_id'] != source['attempt_id'] or source['attempt_id'] != identity['attempt_id'] - or source['request_sha256'] != identity['request_sha256'] - or type(identity['spec_sha256']) is not str or not HASH.fullmatch(identity['spec_sha256'])): - invalid('Run source and custody identities differ.') - encoded = source['custody_receipt_base64'] - if type(encoded) is not str or not 1 <= len(encoded) <= 21848: - invalid('Invalid retained custody receipt encoding.') - try: - receipt_raw = base64.b64decode(encoded, validate=True) - except (ValueError, binascii.Error) as error: - raise WorkbenchError('Scientific fields: Invalid retained custody receipt encoding.') from error - if (not 0 < len(receipt_raw) <= 16384 - or base64.b64encode(receipt_raw).decode('ascii') != encoded - or hashlib.sha256(receipt_raw).hexdigest() != source['custody_receipt_sha256']): - invalid('Retained custody receipt bytes differ.') try: - receipt = recipe_process.validate_receipt(receipt_raw, identity) + recipe_process.validate_artifact_source(source, digest, len(raw)) except recipe_process.ProcessError as error: - raise WorkbenchError('Scientific fields: Invalid retained custody evidence.') from error - if receipt['outcome'] != 'completed' or not receipt['drained']: - invalid('Run source requires completed and drained custody.') + invalid(str(error), error.code, 409 if error.code == 'conflict' else 400) else: invalid('Unsupported scientific source method.') if source['measured_sha256'] != digest: diff --git a/static/workspace-external.js b/static/workspace-external.js index 58f97523..f047cbe8 100644 --- a/static/workspace-external.js +++ b/static/workspace-external.js @@ -3,31 +3,50 @@ // Binding shapes remain byte-compatible: project references never gain a kind key. window.WorkspaceExternalInputs=(()=>{ const esc=value=>window.workspaceEscape(value),hash=value=>/^[a-f0-9]{64}$/.test(value||''); - function choices(project,releases){return { + function choices(project,releases,runs=[]){return { project_reference:(project.data||[]).filter(item=>item.status==='available'&&item.integrity!=='mismatch'&&hash(item.observed_sha256)).map(item=>({id:item.id,label:item.label,bytes:item.bytes,sha256:item.observed_sha256})), - frozen_release:releases.map(item=>({id:item.release_id,label:item.release_id,bytes:item.bytes,sha256:item.sha256,unavailable:item.status!=='available'||!hash(item.sha256),reason:item.reason||item.status})) + frozen_release:releases.map(item=>({id:item.release_id,label:item.release_id,bytes:item.bytes,sha256:item.sha256,unavailable:item.status!=='available'||!hash(item.sha256),reason:item.reason||item.status})), + runs:runs.filter(run=>run.state==='completed'), + run_artifact:runs.filter(run=>run.state==='completed'&&run.status==='receipt_verified').flatMap(run=>run.artifacts.map(item=>({id:item.artifact_name,run_id:run.run_id,label:item.artifact_name+' · '+item.role,bytes:item.bytes,sha256:item.sha256,unavailable:item.status!=='receipt_verified'||!hash(item.sha256)}))) };} - function kind(binding){return binding?.kind==='frozen_release'?'frozen_release':'project_reference';} - function binding(type,item){if(!item||item.unavailable||!hash(item.sha256))throw Error('Choose an available input with an observed hash.');return type==='frozen_release'?{kind:'frozen_release',release_id:item.id,expected_sha256:item.sha256}:{reference_id:item.id,expected_sha256:item.sha256};} - function options(type,value,list,max){ - const id=value?.release_id||value?.reference_id||'',items=list[type]||[]; + function kind(binding){return ['frozen_release','run_artifact'].includes(binding?.kind)?binding.kind:'project_reference';} + function binding(type,item){ + if(!item||item.unavailable||!hash(item.sha256))throw Error('Choose an available input with an observed hash.'); + if(type==='run_artifact')return {kind:'run_artifact',run_id:item.run_id,artifact_name:item.id,expected_sha256:item.sha256}; + return type==='frozen_release'?{kind:'frozen_release',release_id:item.id,expected_sha256:item.sha256}:{reference_id:item.id,expected_sha256:item.sha256}; + } + function options(type,value,list,max,runId=''){ + const id=value?.artifact_name||value?.release_id||value?.reference_id||'',items=(list[type]||[]).filter(item=>type!=='run_artifact'||item.run_id===runId); const retained=id&&!items.some(item=>item.id===id&&item.sha256===value.expected_sha256); - return `${retained?``:''}${items.map(item=>``).join('')}`; + return `${retained?``:''}${items.map(item=>``).join('')}`; + } + function runOptions(value,list){ + const id=value?.run_id||'',runs=list.runs||[]; + return `${id&&!runs.some(run=>run.run_id===id)?``:''}${runs.map(run=>``).join('')}`; } function html(inputs,specs,list){return Object.keys(specs).map(name=>{ const value=inputs[name],type=kind(value),max=specs[name].max_bytes; - return `
Input · ${esc(name)}Maximum ${esc(max)} bytes

${esc(value?.expected_sha256||'Choose an input to bind its exact SHA256.')}

`; + return `
Input · ${esc(name)}Maximum ${esc(max)} bytes

${esc(value?.expected_sha256||'Choose an input to bind its exact SHA256.')}

`; }).join('');} function bind(root,inputs,specs,list,changed){ const find=(attribute,name)=>Array.from(root.querySelectorAll('['+attribute+']')).find(node=>node.getAttribute(attribute)===name); for(const selector of root.querySelectorAll('[data-external-input-kind]')){ - const name=selector.getAttribute('data-external-input-kind'),target=find('data-external-input',name),hint=find('data-external-input-hash',name); - selector.onchange=()=>{target.innerHTML=options(selector.value,null,list,specs[name].max_bytes);hint.textContent='Choose an input to bind its exact SHA256.';changed(name,null);}; - target.onchange=()=>{if(target.selectedOptions?.[0]?.hasAttribute('data-input-retained'))return;const item=list[selector.value]?.find(item=>item.id===target.value);if(target.value===''||!item||item.unavailable||item.bytes>specs[name].max_bytes){changed(name,null);return;}const value=binding(selector.value,item);hint.textContent=value.expected_sha256;changed(name,value);}; + const name=selector.getAttribute('data-external-input-kind'),target=find('data-external-input',name),hint=find('data-external-input-hash',name),run=find('data-external-input-run',name),group=find('data-external-input-run-group',name); + const clear=()=>{target.innerHTML=options(selector.value,null,list,specs[name].max_bytes,run?.value);hint.textContent='Choose an input to bind its exact SHA256.';changed(name,null);}; + selector.onchange=()=>{if(run){run.value='';run.disabled=selector.value!=='run_artifact';run.required=!run.disabled;group.hidden=run.disabled;}clear();}; + if(run)run.onchange=clear; + target.onchange=()=>{if(target.selectedOptions?.[0]?.hasAttribute('data-input-retained'))return;const item=list[selector.value]?.find(item=>item.id===target.value&&(selector.value!=='run_artifact'||item.run_id===run?.value));if(target.value===''||!item||item.unavailable||item.bytes>specs[name].max_bytes){changed(name,null);return;}const value=binding(selector.value,item);hint.textContent=value.expected_sha256;changed(name,value);}; } } - async function candidates(){const response=await (window.TuldokAuth?.fetch||fetch)('/api/workbench/releases/input-candidates');const value=await response.json();if(!response.ok)throw Error(value.error||'Frozen release choices unavailable.');return value.releases;} - return {choices,binding,html,bind,candidates}; + async function candidates(route='/api/workbench/releases/input-candidates',key='releases'){ + const response=await (window.TuldokAuth?.fetch||fetch)(route),value=await response.json(); + if(!response.ok||!Array.isArray(value[key]))throw Error(value.error||'Input choices unavailable.');return value[key]; + } + async function catalogs(){ + const results=await Promise.allSettled([candidates(),candidates('/api/workspace/run-artifact-inputs','runs')]); + return {releases:results[0].status==='fulfilled'?results[0].value:[],runs:results[1].status==='fulfilled'?results[1].value:[],errors:results.flatMap((item,index)=>item.status==='rejected'?[(index?'Run artifacts: ':'Frozen releases: ')+item.reason.message]:[])}; + } + return {choices,binding,html,bind,candidates,catalogs}; })(); window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,getRunId,isDirty,replaceGraph,changeGraph,render,refresh,onRun,onImported}){ @@ -78,15 +97,15 @@ window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,getRunId } function renderPipeline(root,{graph,graphs,busy,choose}){ const execute=graph.nodes.find(node=>node.type==='execute'),input=graph.nodes.find(node=>node.type==='input').parameters; - root.innerHTML=`

Saved recipes

External recipe

${isDirty()?'Unsaved changes':'Saved r'+graph.revision}
1 · Exact inputs2 · Execute captured recipe3 · Verify outputs
${fieldsHTML(execute.parameters,definitions,'external-param-')}

Input bindings

${inputChoices?window.WorkspaceExternalInputs.html(input.inputs||{},inputSpecs,inputChoices):'

Load observed project data and frozen releases to edit exact bindings.

'}
Exact project and input references
${esc(JSON.stringify(input,null,2))}

Execution approval

${permissions}

${preflight?`

Material SHA256 ${esc(preflight.material_sha256)}

Graph r${esc(preflight.graph_revision)} · runtime ${esc(preflight.material?.recipe?.runtime||preflight.runtime?.profile||preflight.runtime?.id||'declared runtime')}

Captured sources, inputs, runtime and limits
${esc(JSON.stringify(preflight.material,null,2))}
Declared output roles
${esc(JSON.stringify(preflight.outputs,null,2))}
`:'

Preflight checks identity without executing project code.

'}
${owner()?`
Grant this material to an actor${grantReceipt?`

Approval ${esc(grantReceipt.id||grantReceipt.approval_id)} · actor ${esc(grantReceipt.consumer_actor_id||recipient)}

`:''}
`:`

The grant must bind your actor, this graph revision and this exact material digest.

`}${pending?`

Request ${esc((pending.body||pending).request_key)} for graph ${esc((pending.body||pending).graph_id)} r${esc((pending.body||pending).graph_revision)} has an unresolved acknowledgement. Originating actor ${esc(pending.actor_id||'unknown')}. Reconcile this same request before starting another.

`:''}

Execution receipts establish process and artifact custody, not scientific correctness.

`; + root.innerHTML=`

Saved recipes

External recipe

${isDirty()?'Unsaved changes':'Saved r'+graph.revision}
1 · Exact inputs2 · Execute captured recipe3 · Verify outputs
${fieldsHTML(execute.parameters,definitions,'external-param-')}

Input bindings

${inputChoices?window.WorkspaceExternalInputs.html(input.inputs||{},inputSpecs,inputChoices):'

Load project data, frozen releases and completed Run artifacts to edit exact bindings.

'}
Exact project and input references
${esc(JSON.stringify(input,null,2))}

Execution approval

${permissions}

${preflight?`

Material SHA256 ${esc(preflight.material_sha256)}

Graph r${esc(preflight.graph_revision)} · runtime ${esc(preflight.material?.recipe?.runtime||preflight.runtime?.profile||preflight.runtime?.id||'declared runtime')}

Captured sources, inputs, runtime and limits
${esc(JSON.stringify(preflight.material,null,2))}
Declared output roles
${esc(JSON.stringify(preflight.outputs,null,2))}
`:'

Preflight checks identity without executing project code.

'}
${owner()?`
Grant this material to an actor${grantReceipt?`

Approval ${esc(grantReceipt.id||grantReceipt.approval_id)} · actor ${esc(grantReceipt.consumer_actor_id||recipient)}

`:''}
`:`

The grant must bind your actor, this graph revision and this exact material digest.

`}${pending?`

Request ${esc((pending.body||pending).request_key)} for graph ${esc((pending.body||pending).graph_id)} r${esc((pending.body||pending).graph_revision)} has an unresolved acknowledgement. Originating actor ${esc(pending.actor_id||'unknown')}. Reconcile this same request before starting another.

`:''}

Execution receipts establish process and artifact custody, not scientific correctness.

`; const $=id=>root.querySelector('#'+id); $('real-graph-select').onchange=event=>choose(event.target.value);$('real-refresh').onclick=refresh; $('external-name').oninput=event=>changeGraph(()=>graph.name=event.target.value,false); $('external-input-choices').onclick=()=>action(async()=>{ const origin=snapshot(); - const [project,definition,releases]=await Promise.all([api('projects/'+enc(input.project_id)),api('projects/'+enc(input.project_id)+'/recipes/'+enc(input.recipe_id)+'?expected_sha256='+enc(input.expected_sha256)),window.WorkspaceExternalInputs.candidates()]); + const [project,definition,catalog]=await Promise.all([api('projects/'+enc(input.project_id)),api('projects/'+enc(input.project_id)+'/recipes/'+enc(input.recipe_id)+'?expected_sha256='+enc(input.expected_sha256)),window.WorkspaceExternalInputs.catalogs()]); if(!current(origin))return; - inputChoices=window.WorkspaceExternalInputs.choices(project,releases);inputSpecs=definition.inputs;status('Loaded exact input choices. Changing a binding requires saving and a new preflight.'); + inputChoices=window.WorkspaceExternalInputs.choices(project,catalog.releases,catalog.runs);inputSpecs=definition.inputs;status(catalog.errors.length?catalog.errors.join(' ')+' Saved bindings remain unchanged.':'Loaded input choices. Changing a binding requires saving and a new preflight.'); }); if(inputChoices)window.WorkspaceExternalInputs.bind(root,input.inputs||{},inputSpecs,inputChoices,(name,value)=>changeGraph(()=>{if(value)input.inputs[name]=value;else delete input.inputs[name];},false)); bindParameters(root,definitions,(name,value)=>changeGraph(()=>execute.parameters[name]=value,false)); @@ -115,7 +134,7 @@ window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,getRunId return `Creates a draft. Review and freeze separately.`; } async function importArtifact(run,artifact){ - if(!importable(run,artifact))throw Error('Only verified scientific fields from a completed external run can be imported.'); + if(!importable(run,artifact))throw Error('Only scientific-field outputs from a completed external run can be imported; the server verifies selected bytes.'); if(!identity()?.scopes.includes('edit'))throw Error('Editing permission is required to import a draft.'); const origin=snapshot(),runId=run.id,name=artifact.name,expected=artifact.sha256; let record; @@ -135,7 +154,7 @@ window.WorkspaceExternal=function({api,action,status,getGraph,getIntent,getRunId const result=run.state==='completed'&&value.result?.status==='hash_verified'?value.result:null; const summary=result?.summary?.format==='tuldok_result_summary_v1'?result.summary:null; const checkpoint=summary?.checkpoint&&(run.artifacts||[]).find(item=>item.name===summary.checkpoint&&item.role==='checkpoint'&&!item.locked); - return `

Recipe progress

${esc(label)}

${progress?`

${esc(progress.stage)} · ${esc(progress.completed)} / ${esc(progress.total)}

${Object.entries(progress.metrics||{}).map(([name,value])=>`
${esc(name)}
${esc(value)}
`).join('')}
`:'

No structured recipe progress reported. Work total and remaining duration are unknown.

'}${summary?`

${esc(summary.title)}

Hash-verified result artifact · recipe-reported values, not scientific qualification.

${summary.metrics.map(item=>`
${esc(item.name)}
${esc(item.value)} ${esc(item.unit)}
`).join('')}
    ${summary.scope.map(item=>`
  • ${esc(item)}
  • `).join('')}

SHA256 ${esc(result.sha256)}

${checkpoint?`Download checkpoint`:''}
`:value.result?.status==='unavailable'?'

Result summary unavailable. Inspect verified artifacts for retained evidence.

':''}${['stdout','stderr'].map(name=>{const stream=(verified||live)&&value.streams?.[name];return `
${name} · ${stream?esc(stream.available_bytes)+' bytes'+(stream.truncated?' · bounded tail':''):'unavailable'}${stream?`

${esc(label)} · starts at byte ${esc(stream.start_byte)}

${esc(stream.text)}
`:'

No readable stream evidence.

'}
`;}).join('')}
`; + return `

Recipe progress

${esc(label)}

${progress?`

${esc(progress.stage)} · ${esc(progress.completed)} / ${esc(progress.total)}

${Object.entries(progress.metrics||{}).map(([name,value])=>`
${esc(name)}
${esc(value)}
`).join('')}
`:'

No structured recipe progress reported. Work total and remaining duration are unknown.

'}${summary?`

${esc(summary.title)}

Hash-verified result artifact · recipe-reported values, not scientific qualification.

${summary.metrics.map(item=>`
${esc(item.name)}
${esc(item.value)} ${esc(item.unit)}
`).join('')}
    ${summary.scope.map(item=>`
  • ${esc(item)}
  • `).join('')}

SHA256 ${esc(result.sha256)}

${checkpoint?`Download checkpoint`:''}
`:value.result?.status==='unavailable'?'

Result summary unavailable. Inspect recorded outputs for retained evidence.

':''}${['stdout','stderr'].map(name=>{const stream=(verified||live)&&value.streams?.[name];return `
${name} · ${stream?esc(stream.available_bytes)+' bytes'+(stream.truncated?' · bounded tail':''):'unavailable'}${stream?`

${esc(label)} · starts at byte ${esc(stream.start_byte)}

${esc(stream.text)}
`:'

No readable stream evidence.

'}
`;}).join('')}
`; } function runHTML(run,progress){return `
${progressHTML(run,progress)}

Trusted local execution · Custody: ${esc(run.custody_phase||'unconfirmed')}

${run.state==='unknown'?'

Process cessation is unconfirmed. Same-boot execution may remain fenced; acknowledgement alone cannot clear custody.

':''}
Custody and approved material

Approval ${esc(run.approval_id)} · actor ${esc(run.actor_id)}

Material SHA256 ${esc(run.material_sha256)}

${permissions} Completion is not scientific qualification.

${esc(JSON.stringify(run.material,null,2))}
`;} window.addEventListener('tuldok-session',()=>{invalidate(true);if(isGraph(getGraph()))render();}); @@ -169,7 +188,7 @@ window.WorkspaceExternalProject=function({onCreated,announce}){ return {cancel,async open(detail,item){ if(busy)return;const epoch=++generation;project=copy(detail);reference=copy(item);definition=null;bindings={};choicesError='';inputChoices=null; dialog.innerHTML='

Inspecting recipe definition…

Reading declared fields. No project code is executed.

';dialog.querySelector('button').onclick=cancel;dialog.showModal(); - try{const value=await api('projects/'+enc(detail.id)+'/recipes/'+enc(item.id)+'?expected_sha256='+enc(item.observed_sha256));if(epoch!==generation||!dialog.open)return;if(!value.definition||!value.inputs||!value.parameter_definitions)throw Error('Unsupported external recipe definition.');definition=value;let releases=[];try{releases=await window.WorkspaceExternalInputs.candidates();}catch(error){choicesError=error.message+' Project references remain available.';}if(epoch!==generation||!dialog.open)return;inputChoices=window.WorkspaceExternalInputs.choices(project,releases);render();dialog.querySelector('#external-create-name').focus();} + try{const value=await api('projects/'+enc(detail.id)+'/recipes/'+enc(item.id)+'?expected_sha256='+enc(item.observed_sha256));if(epoch!==generation||!dialog.open)return;if(!value.definition||!value.inputs||!value.parameter_definitions)throw Error('Unsupported external recipe definition.');definition=value;const catalog=await window.WorkspaceExternalInputs.catalogs();choicesError=catalog.errors.join(' ');if(epoch!==generation||!dialog.open)return;inputChoices=window.WorkspaceExternalInputs.choices(project,catalog.releases,catalog.runs);render();dialog.querySelector('#external-create-name').focus();} catch(error){if(epoch===generation&&dialog.open){dialog.querySelector('p').textContent=error.message;}} }}; function copy(value){return structuredClone(value);} diff --git a/static/workspace-real.js b/static/workspace-real.js index 5c69d1a6..dce78cb9 100644 --- a/static/workspace-real.js +++ b/static/workspace-real.js @@ -285,7 +285,7 @@ rowsError = error.message; } } - if(detail&&external.isGraph(detail.graph)&&!detail.artifact_status){ + if(detail&&external.isGraph(detail.graph)&&detail.artifact_status?.status!=='unavailable'){ try{progress=await api('runs/'+encoded(id)+'/progress');} catch{progress=null;} } @@ -320,7 +320,7 @@ ${!externalRun && !integrity && (run.current_test_exposure?.exposed || run.prior_test_exposure?.exposed) ? '

This release contains previously exposed/reused test material. Results cannot be presented as fresh holdout evidence.

' : ""} ${externalRun ? external.runHTML(run,runProgress) : integrity ? integritySummaryHTML(run.graph, run.validation_summary) : ""} ${metrics ? `
${(metrics.accuracy * 100).toFixed(1)}%Validation accuracy
${metrics.examples}Validation examples
${metrics.macro_f1.toFixed(3)}Validation macro F1
` : ""} -
${!externalRun?`

Stage evidence

${(run.node_attempts || []).map((stage, i) => `
${i + 1}${esc(stage.type === "train" ? trainingTitle(run.graph) : (integrity ? {release: "Frozen release", validate: `Validate ${integrity.name.toLowerCase()} integrity`, publish: "Publish integrity report"}[stage.type] : titles[stage.type]) || stage.type)}${esc(stage.state)}
`).join("")}
Bounded progress log
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}
`:`
Run lifecycle events
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}
`}

Verified artifacts

${run.artifacts.length ? `
    ${run.artifacts.map(a => `
  • ${a.locked ? `${esc(a.name)} · test access locked` : `${esc(a.name)}`}${a.role ? esc(a.role) + " · " : ""}${a.bytes.toLocaleString()} bytes · SHA ${esc(short(a.sha256))}${external.artifactActionHTML(run,a)}
  • `).join("")}
` : '

No published artifacts yet.

'}

Every artifact read verifies its frozen hash.

+
${!externalRun?`

Stage evidence

${(run.node_attempts || []).map((stage, i) => `
${i + 1}${esc(stage.type === "train" ? trainingTitle(run.graph) : (integrity ? {release: "Frozen release", validate: `Validate ${integrity.name.toLowerCase()} integrity`, publish: "Publish integrity report"}[stage.type] : titles[stage.type]) || stage.type)}${esc(stage.state)}
`).join("")}
Bounded progress log
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}
`:`
Run lifecycle events
${esc((run.events || []).map(e => `${e.at} [${e.state}] ${e.message}`).join("\n"))}
`}

${run.artifact_status?.status==='receipt_verified'?'Recorded outputs; verified on download':'Verified artifacts'}

${run.artifacts.length ? `
    ${run.artifacts.map(a => `
  • ${a.locked ? `${esc(a.name)} · test access locked` : `${esc(a.name)}`}${a.role ? esc(a.role) + " · " : ""}${a.bytes.toLocaleString()} bytes · SHA ${esc(short(a.sha256))}${external.artifactActionHTML(run,a)}
  • `).join("")}
` : '

No published artifacts yet.

'}

${run.artifact_status?.status==='receipt_verified'?'Output receipt verified. Payload bytes are checked individually when requested.':'Every artifact read verifies its frozen hash.'}

${run.state === "completed" && !externalRun && !integrity ? `

Per-example predictions

${esc(split)}${split === "test" ? " · exposed" : ""}
${predictionsHTML()}
` : ""}
Frozen identity, source provenance and parameters
${esc(JSON.stringify({ dataset_id: run.dataset_id, run_id: run.id, release_sha256: run.release_sha256, graph_sha256: run.graph_sha256, graph: run.graph, adapter: run.adapter, runtime: run.runtime, parameters: run.parameters, prior_test_exposure: run.prior_test_exposure, current_test_exposure: run.current_test_exposure, test_exposure: run.test_exposure }, null, 2))}
`}`; }); diff --git a/tests/browser_workspace_rheon_evaluation.cjs b/tests/browser_workspace_rheon_evaluation.cjs new file mode 100644 index 00000000..19e5eb6e --- /dev/null +++ b/tests/browser_workspace_rheon_evaluation.cjs @@ -0,0 +1,93 @@ +'use strict'; +// Actual original autonomous evaluation with explicit orchestration; no authored/model fallback. +const fixtureAuth=require('./auth_helpers.cjs').authenticatedFixture(),fetch=fixtureAuth.fetch; +const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); +const {spawn,execFileSync}=require('node:child_process'); +const {pageLoadTracker}=require('./browser_page_load.cjs'); +const {waitForDebugger}=require('./browser_startup.cjs'); +const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); +const root=path.resolve(__dirname,'..'),report=qaDirectory(root,'workspace-rheon-evaluation'); +function option(name,environment){const index=process.argv.indexOf(name);return index<0?process.env[environment]:process.argv[index+1];} +const catalog=option('--catalog','TULDOK_CPU_CATALOG'),projectRoot=option('--project','TULDOK_CPU_PROJECT'),snapshot=option('--snapshot','TULDOK_CPU_SNAPSHOT'); +assert.ok(catalog&&projectRoot&&/^[a-f0-9]{64}$/.test(snapshot||''),'Explicit precreated --catalog, --project and --snapshot required'); +assert.ok(process.argv.includes('--runtime-qualified'),'Pass --runtime-qualified only after the captured runtime qualification passes'); +const sourceRun=option('--source-run','TULDOK_EVALUATION_SOURCE_RUN');assert.match(sourceRun||'',/^[a-f0-9]{32}$/); +const numpy=process.env.TULDOK_TRAINING_PYTHON||'/workspace/Tuldok/.venv-training/bin/python'; +const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-rheon-evaluation-browser-')); +const project=JSON.parse(fs.readFileSync(path.join(projectRoot,'project.json'))); +const sha=raw=>crypto.createHash('sha256').update(raw).digest('hex'); +const originalFiles=Object.fromEntries(fs.readdirSync(projectRoot).filter(name=>fs.statSync(path.join(projectRoot,name)).isFile()).map(name=>[name,sha(fs.readFileSync(path.join(projectRoot,name)))])); +const inspections=[],captures=[],checks=[],requests=[],errors=[],children=[],tracker=pageLoadTracker();const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms));let ws,send,evaluate,serverErrors=''; +async function until(fn,message='Timed out',seconds=60){for(let i=0;i['-u','app.py','--port',String(port),'--data',catalog,'--project-root','sample='+projectRoot]; +(async()=>{ + let server=launch('python3',serverArgs(0),{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); + const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); + const base='http://127.0.0.1:'+port; + await fixtureAuth.register('http://127.0.0.1:'+port,catalog,root); + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); + const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); + ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next,timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP timeout '+method));},20000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});ws.send(JSON.stringify({id,method,params}));}); + ws.onmessage=event=>{const message=JSON.parse(event.data);tracker.observe(message);if(message.id){const task=pending.get(message.id);pending.delete(message.id);if(task)message.error?task.reject(message.error):task.resolve(message.result);}if(message.method==='Runtime.exceptionThrown')errors.push(message.params.exceptionDetails);if(message.method==='Network.requestWillBeSent')requests.push({url:message.params.request.url,method:message.params.request.method});}; + evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + const doc=frame=>frame?'document.getElementById("catalog-frame").contentDocument':'document'; + const element=(selector,frame=false)=>`${doc(frame)}.querySelector(${JSON.stringify(selector)})`,text=(selector,frame=false)=>evaluate(`${element(selector,frame)}?.textContent`); + async function click(selector,frame=false){ + const box=await until(()=>evaluate(`(async()=>{const e=${element(selector,frame)};if(!e||e.disabled||!e.getClientRects().length)return null;${frame?'document.getElementById("catalog-frame").scrollIntoView({block:"start"});':''}e.scrollIntoView({block:'center'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));const b=e.getBoundingClientRect(),x=b.x+b.width/2,y=b.y+b.height/2,hit=${doc(frame)}.elementFromPoint(x,y),f=${frame?'document.getElementById("catalog-frame").getBoundingClientRect()':'{x:0,y:0}'};return hit&&(hit===e||e.contains(hit))?{x:x+f.x,y:y+f.y}:null})()`),'Click target unavailable '+selector); + for(const type of ['mousePressed','mouseReleased'])await send('Input.dispatchMouseEvent',{type,button:'left',clickCount:1,...box}); + } + async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Escape:27,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} + async function fill(selector,value,frame=false){await click(selector,frame);await key('a','KeyA',2);await send('Input.insertText',{text:value});} + const select=(selector,value,frame=false)=>evaluate(`(()=>{const e=${element(selector,frame)};e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); + async function nav(view){await evaluate(`document.getElementById('nav-${view}').focus()`);await key('Enter');await until(()=>evaluate(`!document.getElementById('view-${view}').hidden && !document.getElementById('project-${view}-root').hidden`));await until(()=>text(`#project-${view}-root`).then(value=>value?.includes('PROJECT · INSPECTION ONLY')));} + const check=value=>{checks.push(value);console.log('PASS',value);}; + async function reload(){const previous=(await send('Page.getFrameTree')).frameTree.frame;await send('Page.reload');await until(()=>tracker.reloaded(previous),'Main document reload not observed');} + async function openProject(manifest,name,id){await until(()=>evaluate('typeof document.getElementById("open-project")?.onclick==="function"'));await click('#open-project');await until(()=>evaluate('document.getElementById("project-open-dialog").open && !document.getElementById("project-open-submit").disabled'));await select('#project-root','sample');await fill('#project-manifest',manifest);await evaluate('document.getElementById("project-open-submit").focus()');await key('Enter');await until(()=>evaluate('document.getElementById("dataset").value==='+JSON.stringify('project:'+id)));await nav('data');assert.match(await text('#project-data-root'),new RegExp(name.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')));} + await send('Page.enable');await send('Page.setLifecycleEventsEnabled',{enabled:true});await send('Runtime.enable');await send('Network.enable'); + const offlineCanary=process.env.LD_PRELOAD?JSON.parse(execFileSync('python3',['-c',"import sys;sys.path.insert(0,'tests');from test_workspace_offline import CANARY;exec(CANARY)"],{cwd:root,encoding:'utf8'})):null; + await fixtureAuth.browser(send,'http://127.0.0.1:'+port); + await send('Page.navigate',{url:base+'/workspaces'});await until(()=>evaluate('!!document.getElementById("open-project")')); + async function captureWorkspace(name,width,selector){await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});await evaluate(`${element(selector)}.scrollIntoView({block:'start'})`);await pause(100);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'));const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Actual autonomous CPU evaluation; previously exposed seeds, no GPU or new holdout claim'});} + async function api(route,body){const response=await fetch(base+route,body===undefined?{}:{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});const value=await response.json();assert.ok(response.ok,JSON.stringify({route,status:response.status,value}));return value;} + async function artifact(run,name){const item=run.artifacts.find(item=>item.name===name);assert.ok(item,name);const response=await fetch(base+item.url);assert.equal(response.status,200);const bytes=Buffer.from(await response.arrayBuffer());assert.equal(sha(bytes),item.sha256);return bytes;} + async function completed(id){let logged=0;return until(async()=>{const run=await api('/api/workspace/runs/'+id);assert.ok(!['failed','unknown','cancelled','interrupted'].includes(run.state),JSON.stringify({state:run.state,error:run.error,events:run.events}));if(Date.now()-logged>30000){console.log('CPU run',run.id,run.state,run.custody_phase);logged=Date.now();}return run.state==='completed'?run:null;},'Actual approved evaluation did not complete',2700);} + const prior=await api('/api/workspace/runs/'+sourceRun);assert.equal(prior.state,'completed'); + const bindings={checkpoint:'model.pt',freeze:'freeze.json',training_inputs:'input-receipt.json',training_result:'result.json'}; + for(const name of Object.values(bindings))assert.ok(prior.artifacts.find(item=>item.name===name)); + await openProject('project.json',project.name,project.id);await nav('pipeline'); + await click('[data-project-recipe="evaluate"]');await until(()=>evaluate('!!document.querySelector("[data-external-input=checkpoint]")')); + for(const [slot,name] of Object.entries(bindings)){await select('[data-external-input-kind="'+slot+'"]','run_artifact'); + await select('[data-external-input-run="'+slot+'"]',sourceRun);await select('[data-external-input="'+slot+'"]',name);} + await select('[data-external-input="build_receipt"]','build_receipt');await fill('#external-create-name','Autonomous CPU evaluation · exposed seeds 900–903'); + await evaluate('document.getElementById("external-create-submit").focus()');await key('Enter');await until(()=>evaluate('!!document.getElementById("external-preflight")')); + await click('#external-preflight');await until(()=>evaluate('!document.getElementById("external-reviewed").disabled'),'Evaluation preflight unavailable',180); + await captureWorkspace('evaluation-pipeline-1440',1440,'#external-material');await captureWorkspace('evaluation-pipeline-390',390,'#external-material'); + const previousRunIds=new Set((await api('/api/workspace/runs')).map(run=>run.id));await click('#external-reviewed');await evaluate('document.getElementById("external-submit").focus()');await key('Enter'); + const submitted=await until(async()=>{const runs=await api('/api/workspace/runs');return runs.find(run=>run.execution_kind==='external_recipe'&&!previousRunIds.has(run.id));}); + fs.writeFileSync(path.join(report,'submitted-run.json'),JSON.stringify(submitted,null,2)); + let lastLog=0,lastProgress=0,liveCaptured=false;const observations=[]; + const run=await until(async()=>{const current=await api('/api/workspace/runs/'+submitted.id);if(['failed','unknown','cancelled','interrupted'].includes(current.state)){fs.writeFileSync(path.join(report,'failed-run.json'),JSON.stringify(current,null,2));throw Error(JSON.stringify({id:current.id,state:current.state,custody_phase:current.custody_phase,error:current.error,events:current.events}));}if(Date.now()-lastProgress>5000){lastProgress=Date.now();const progress=await api('/api/workspace/runs/'+current.id+'/progress');observations.push(progress);fs.writeFileSync(path.join(report,'progress-observations.json'),JSON.stringify(observations,null,2));if(progress.progress&&!liveCaptured){await until(()=>evaluate('!!document.getElementById("external-progress")'));await captureWorkspace('evaluation-live-1440',1440,'#external-progress');await captureWorkspace('evaluation-live-390',390,'#external-progress');liveCaptured=true;}if(Date.now()-lastLog>30000){console.log('Autonomous evaluation',current.id,current.state,current.custody_phase,progress.progress?.stage,progress.progress?.completed);lastLog=Date.now();}}return current.state==='completed'?current:null;},'Original full evaluation did not complete',2700); + fs.writeFileSync(path.join(report,'completed-run.json'),JSON.stringify(run,null,2));assert.equal(run.artifacts.length,89);assert.ok(JSON.stringify(run.material).includes(snapshot)); + const finalProgress=await api('/api/workspace/runs/'+run.id+'/progress');assert.equal(finalProgress.result.status,'hash_verified');assert.equal(finalProgress.evidence,'custody_verified');fs.writeFileSync(path.join(report,'completed-progress.json'),JSON.stringify(finalProgress,null,2)); + await until(()=>text('#real-run-state').then(value=>value==='completed'));await until(()=>evaluate('!!document.getElementById("external-checkpoint-download")')); + await captureWorkspace('evaluation-results-1440',1440,'#external-result-summary');await captureWorkspace('evaluation-results-390',390,'#external-result-summary'); + async function streamInventory(current,label){const inventory=[];for(const item of current.artifacts){const response=await fetch(base+item.url);assert.equal(response.status,200);assert.equal(Number(response.headers.get('content-length')),item.bytes);let bytes=0;const hash=crypto.createHash('sha256');for await(const chunk of response.body){bytes+=chunk.length;hash.update(chunk);}const digest=hash.digest('hex');assert.equal(bytes,item.bytes);assert.equal(digest,item.sha256);inventory.push({name:item.name,bytes,sha256:digest});if(inventory.length%8===0)console.log(label,'verified HTTP artifacts',inventory.length,'/',current.artifacts.length);}fs.writeFileSync(path.join(report,label+'-http-inventory.json'),JSON.stringify(inventory,null,2));return inventory;} + const inventory=await streamInventory(run,'completed');check('All89 completed outputs independently streamed and hash-verified via authenticated HTTP without a third retained copy'); + const localArtifacts=path.join(catalog,'workspace-runs',run.id);const cold=JSON.parse(execFileSync(numpy,['tests/qualify_rheon_evaluation.py','--artifacts',localArtifacts,'--http-inventory',path.join(report,'completed-http-inventory.json'),'--report',path.join(report,'cold-report.json')],{cwd:root,encoding:'utf8',maxBuffer:8*1024*1024})); + check('Independent owner-local cold audit verifies every six-field array/wire byte, original NPZ overlap, all-frame errors and benchmark arithmetic'); + const representative='fields-learned-900-000.zip';await click('[data-import-run-artifact="'+representative+'"]');await until(()=>evaluate(`${doc(true)}?.getElementById('scientific-inspection')?.hidden===false`));await until(()=>text('#scientific-status',true).then(value=>value?.includes('Producer-declared stage:'))); + const imported=(await api('/api/workbench/records')).items.find(item=>item.content_hash===run.artifacts.find(item=>item.name===representative).sha256);assert.ok(imported);assert.equal(imported.review,'draft'); + await select('#scientific-field','pressure',true);await select('#scientific-axis','z',true);await fill('#scientific-note','QA owner inspected representative retained autonomous evaluation chunk after all-array native wire/NPZ audit. Previously exposed seed900; not new holdout or general physics acceptance.',true);await select('#record-review','human_reviewed',true);await evaluate(`${element('#editor button:not([type="button"])',true)}.focus()`);await key('Enter');await until(async()=>(await api('/api/workbench/records/'+imported.id)).review==='human_reviewed'); + const inspection=await api('/api/workbench/records/'+imported.id+'/scientific?frame=0&field=pressure&axis=z&index=0');fs.writeFileSync(path.join(report,'representative-inspection.json'),JSON.stringify(inspection,null,2)); + for(const width of [1440,390]){await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});await evaluate(`(async()=>{const f=document.getElementById('catalog-frame');f.scrollIntoView({block:'start'});f.contentDocument.getElementById('scientific-inspection').scrollIntoView({block:'start'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));window.scrollBy(0,f.getBoundingClientRect().top)})()`);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'));assert.ok(await evaluate(`${doc(true)}.documentElement.scrollWidth<=${doc(true)}.defaultView.innerWidth+1`));const file='evaluation-data-'+width+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Representative actual retained evaluation field chunk, QA owner review'});} + check('One representative complete scientific chunk imports into Data and receives explicit QA-owner inspection review'); + server.kill('SIGINT');await until(()=>server.exitCode!==null);server=launch('python3',serverArgs(port),{cwd:root,stdio:['ignore','pipe','pipe']});let reopened='';server.stdout.on('data',chunk=>reopened+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk);await until(()=>reopened.includes('Tuldok:'));await reload(); + const retained=await api('/api/workspace/runs/'+run.id);assert.equal(retained.state,'completed');assert.equal(retained.material_sha256,run.material_sha256);assert.deepEqual(await streamInventory(retained,'reopened'),inventory); + assert.equal((await api('/api/workbench/records/'+imported.id)).review,'human_reviewed');assert.deepEqual(errors,[]);assert.ok(!serverErrors.includes('Traceback'),serverErrors);check('Cold restart preserves all89 HTTP payload hashes and representative reviewed Data chunk'); + fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({source_commit:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),dirty:!!execFileSync('git',['status','--porcelain'],{cwd:root,encoding:'utf8'}).trim(),qualification:'Original autonomous evaluation numerical algorithms with explicit provenance/transport/retention orchestration',previously_exposed_seeds:[900,901,902,903],new_holdout_acceptance:false,gpu_acceptance:false,checks,captures,run_id:run.id,source_training_run:sourceRun,record_id:imported.id,material_sha256:run.material_sha256,runtime_snapshot_sha256:snapshot,cold,offline_canary:offlineCanary,browser:(await send('Browser.getVersion')).product},null,2));console.log('PASS autonomous CPU evaluation browser:',report); +})().catch(async error=>{console.error(error);console.error('Application stderr:',serverErrors);if(evaluate)try{console.error(await evaluate('JSON.stringify({url:location.href,body:document.body.innerText.slice(-6000)})'));}catch{}process.exitCode=1;}).finally(()=>{ws?.close();for(const child of children.reverse())child.kill('SIGTERM');}); diff --git a/tests/qualify_rheon_evaluation.py b/tests/qualify_rheon_evaluation.py new file mode 100644 index 00000000..ba66b692 --- /dev/null +++ b/tests/qualify_rheon_evaluation.py @@ -0,0 +1,245 @@ +"""Independent all-frame wire/NPY/NPZ and numerical audit; never evaluates a model.""" +import argparse +import base64 +from collections import Counter +import hashlib +import io +import json +import math +from pathlib import Path +import statistics +import struct + +import numpy as np + +FIELDS = ( + ('rhs','=0 and tracer.max()<=1,'Tracer bounds fail') + integral=float(np.sum(tracer)/32**3) + require(math.isclose(integral,report['tracer_integral'],rel_tol=1e-12,abs_tol=1e-15),'Independent tracer integral differs') + for key,limit in [('relative_residual',1e-8),('courant',1)]: + require(type(report[key]) in (int,float) and math.isfinite(report[key]) and 0<=report[key]<=limit,'Reported physics gate fails') + return maximum + + +def compare(truth,predicted): + # Preserve the original arithmetic precision deliberately: denominator norm + # and tracer subtraction are float32, centered pressure is float64. + def relative(a,b):return float(np.linalg.norm(a.astype(np.float64)-b)/max(np.linalg.norm(b),1e-30)) + tv=np.concatenate([truth[name].reshape(-1) for name in ('velocity_x','velocity_y','velocity_z')]) + pv=np.concatenate([predicted[name].reshape(-1) for name in ('velocity_x','velocity_y','velocity_z')]) + tp=truth['pressure'].reshape(-1);pp=predicted['pressure'].reshape(-1) + return {'relative_velocity_l2':relative(pv,tv),'relative_tracer_l2':relative(predicted['tracer'].reshape(-1),truth['tracer'].reshape(-1)), + 'tracer_rmse':float(np.sqrt(np.mean((predicted['tracer'].reshape(-1)-truth['tracer'].reshape(-1))**2))), + 'centered_relative_pressure_l2':relative(pp-pp.mean(),tp-tp.mean())} + + +def benchmark(result,freeze): + require(freeze['test_seeds']==[900,901,902,903] and freeze['test_frames']==120 and freeze['benchmark_repetitions']==5,'Frozen evaluation controls changed') + methods=['jacobi','sgs','learned'];require(freeze['methods']==methods,'Frozen methods changed') + expected=[] + for repetition in range(5): + for index,seed in enumerate(range(900,904)): + offset=(repetition+index)%3 + expected.extend((repetition,seed,method) for method in methods[offset:]+methods[:offset]) + records=result['benchmark'];require([(row['repetition'],row['seed'],row['method']) for row in records]==expected,'Incomplete or reordered original benchmark') + for row in records: + require(row['frames']==120 and len(row['reports'])==120,'Missing benchmark frames') + require(math.isfinite(row['wall_ms_including_native_startup_teardown']) and row['wall_ms_including_native_startup_teardown']>0,'Invalid wall timing') + for index,report in enumerate(row['reports'],1): + require(report['frame']==index and math.isclose(report['time'],index/60,rel_tol=1e-12,abs_tol=1e-12),'Benchmark frame/time drift') + for key,limit in [('relative_residual',1e-8),('actual_divergence_max',1e-5),('courant',1)]: + require(type(report[key]) in (int,float) and math.isfinite(report[key]) and 0<=report[key]<=limit,'Benchmark diagnostic fails') + if row['method']=='learned':require(report['iterations']==0,'Learned benchmark used iterative correction') + medians={method:statistics.median(row['wall_ms_including_native_startup_teardown'] for row in records if row['method']==method) for method in methods} + require(medians==result['median_120_frame_wall_ms'],'Incorrect original median arithmetic') + for method in ('jacobi','sgs'):require(result['speedup_vs_'+method]==medians[method]/medians['learned'],'Incorrect original timing ratio') + return {'trajectories':60,'frames':7200,'median_120_frame_wall_ms':medians} + + +def trajectory(directory,method,seed,npz,checks,identity): + import zipfile + index=json.loads((directory/f'wire-{method}-{seed}.json').read_text()) + require((index['method'],index['seed'],index['frames'])==(method,seed,120) and len(index['chunks'])==8,'Wrong trajectory index') + require(index['identity']==identity,'Trajectory identity differs from outer receipt') + stream_hash=hashlib.sha256();offset=0;ready=None + family={'namespace':'rheon-trajectory-probe-scene-v1','id':digest(json.dumps({'scene':'rheon-trajectory-probe-source-v1','grid':32,'seed':seed},sort_keys=True,separators=(',',':')).encode())} + for number,chunk in enumerate(index['chunks']): + first=number*16+1;count=min(16,121-first);name=f'fields-{method}-{seed}-{number:03d}.zip' + require(chunk['path']==name and chunk['frame_start']==first and chunk['frame_end']==first+count-1 and chunk['stdout_start']==offset,'Overlapping/missing chunk/frame ranges') + require(file_digest(directory/name)==(chunk['bytes'],chunk['sha256']),'Chunk index custody mismatch') + with zipfile.ZipFile(directory/name) as archive: + manifest=json.loads(archive.read('manifest.json'));trace=json.loads(archive.read('sources/evaluation-trace.json'));wire=archive.read('sources/wire.bin') + require(manifest['family']==family and len(manifest['frames'])==count,'Wrong protected trajectory family') + require(manifest['grid']=={'shape_xyz':[32]*3,'origin_xyz':[0,0,0],'spacing_xyz':[1/32]*3,'orientation':[[1,0,0],[0,1,0],[0,0,1]],'units':'native length units (unspecified)'},'Scientific grid drift') + require('unspecified' in manifest['time_units'],'Invented time units') + require(trace['identity']==index['identity'] and trace['method']==method and trace['seed']==seed,'Trace identity differs') + for key in ('frame_start','frame_end','stdout_start','stdout_end','stdout_sha256'):require(trace[key]==chunk[key],'Trace range/hash mismatch') + require(trace['includes_ready']==(number==0) and trace['learned_rhs_request_before_report']==(method=='learned'),'Wire protocol metadata mismatch') + require(len(wire)==chunk['stdout_end']-offset and digest(wire)==chunk['stdout_sha256'],'Wire segment identity mismatch') + for source in manifest['sources']: + raw=archive.read(source['path']);require(len(raw)==source['bytes'] and digest(raw)==source['sha256'],'Embedded source custody mismatch') + observed_ready,frames=decode_segment(wire,method,seed,first,count) + if number==0:ready=observed_ready + require(trace['ready']==ready,'Repeated READY declarations differ') + stream_hash.update(wire);offset+=len(wire) + require(trace['reports']==[frame[0] for frame in frames],'Wire reports differ from trace') + require(len(manifest['fields'])==6,'Wrong field inventory') + for fi,(report,arrays,extra_rhs) in enumerate(frames): + frame=manifest['frames'][fi];require(float(frame['time'])==report['time'] and len(frame['arrays'])==6,'Frame metadata drift') + for name,dtype,shape,center,stage in FIELDS: + field=next(item for item in manifest['fields'] if item['id']==name) + require((field['dtype'],field['centering'],field['stage'])==(dtype,center,stage),'Field declaration drift') + item=next(item for item in frame['arrays'] if item['field']==name);raw=archive.read(item['path']) + require(len(raw)==item['bytes'] and digest(raw)==item['sha256'] and raw[:8]==b'\x93NUMPY\x01\x00','NPY identity/version mismatch') + value=np.load(io.BytesIO(raw),allow_pickle=False) + require(value.dtype.str==dtype and value.shape==shape and value.flags.c_contiguous,'NPY dtype/order/shape mismatch') + require(value.tobytes()==arrays[name].tobytes(),'NPY bytes differ from original native stdout') + if name!='rhs': + npz_name={'velocity_x':'velocity_0','velocity_y':'velocity_1','velocity_z':'velocity_2'}.get(name,name) + original=npz[npz_name][first+fi-1] + require(original.dtype==value.dtype and original.tobytes()==value.tobytes(),'Original NPZ and retained complete field differ') + if extra_rhs is not None:require(extra_rhs==arrays['rhs'].tobytes(),'Learned request RHS differs from accepted-frame RHS') + maximum=physics(report,arrays) + if method=='learned':require(report['iterations']==0,'Learned accuracy used iterative correction') + checks.append({'method':method,'seed':seed,'frame':first+fi,'fields':6,'divergence':maximum,'wire_rhs_request_preserved':extra_rhs is not None}) + yield report,arrays + require(offset==index['stdout_bytes'] and stream_hash.hexdigest()==index['stdout_sha256'],'Complete disjoint stdout reconstruction fails') + + +def audit(directory,http_inventory): + directory=Path(directory);inventory=json.loads(Path(http_inventory).read_text()) + require(len(inventory)==89 and len({item['name'] for item in inventory})==89,'Expected89 distinct HTTP outputs') + for item in inventory: + require(Path(item['name']).name==item['name'],'Nonflat artifact') + require(file_digest(directory/item['name'])==(item['bytes'],item['sha256']),'Owner-local artifact differs from HTTP verified bytes') + measured={item['name']:item for item in inventory} + freeze_raw=(directory/'input-freeze.json').read_bytes();freeze=json.loads(freeze_raw) + result=json.loads((directory/'result.json').read_text()) + require(result['schema']=='rheon-trajectory-result-v1' and result['freeze_sha256']==digest(freeze_raw),'Original result/freeze mismatch') + require(result['checkpoint_sha256']==freeze['checkpoint_sha256']==measured['input-model.pt']['sha256'],'Original checkpoint identity mismatch') + for name,expected in result['artifact_sha256'].items():require(measured[name]['sha256']==expected,'Original output hash map differs') + outer=json.loads((directory/'evaluation-receipt.json').read_text()) + require(outer['schema_version']==1 and outer['evaluation']=='fresh_cpu_autonomous_evaluation','Wrong outer evaluation receipt') + require(set(outer['outputs'])==set(measured)-{'evaluation-receipt.json'},'Outer output inventory incomplete') + for name,item in outer['outputs'].items():require(item['sha256']==measured[name]['sha256'] and item['bytes']==measured[name]['bytes'],'Outer output identity mismatch') + identity=outer['identity'];require(identity['checkpoint_sha256']==result['checkpoint_sha256'] and identity['freeze_sha256']==digest(freeze_raw),'Outer input identities differ') + for key in ('runtime_snapshot_sha256','binary_sha256','source_sha256'):require(identity[key]==freeze['source'][key],'Captured source/runtime/native identity differs') + evidence=outer['input_evidence'];require(set(evidence)=={'checkpoint','freeze','training_inputs','training_result','build_receipt'},'Wrong input receipt slots') + slots={'checkpoint':('input-model.pt','checkpoint'),'freeze':('input-freeze.json','training_freeze'),'training_inputs':('input-training-receipt.json','training_inputs'),'training_result':('input-training-result.json','training_result')} + for slot,(name,role) in slots.items(): + require(evidence[slot]['kind']=='run_artifact','Evaluation did not directly consume prior Run artifacts') + source=evidence[slot]['source'];require(source['measured_sha256']==measured[name]['sha256'] and source['artifact_bytes']==measured[name]['bytes'] and source['artifact_role']==role,'Training artifact typed receipt differs') + require({key:source[key] for key in identity['training_origin']}==identity['training_origin'],'Inputs have different originating training Runs') + raw=base64.b64decode(source['custody_receipt_base64'],validate=True);require(digest(raw)==source['custody_receipt_sha256'],'Training custody receipt hash mismatch') + custody=json.loads(raw);require(custody['outcome']=='completed' and custody['drained'] and custody['echild'] and custody['go_received'],'Training custody did not complete') + require(all(custody[key]==value for key,value in source['custody_identity'].items()),'Original training custody identity mismatch') + training=json.loads((directory/'input-training-result.json').read_text());inputs=json.loads((directory/'input-training-receipt.json').read_text()) + require(training['checkpoint_sha256']==result['checkpoint_sha256'] and training['freeze_sha256']==digest(freeze_raw) and training['input_receipt_sha256']==measured['input-training-receipt.json']['sha256'],'Prior training payload hashes disagree') + require(training['releases']==inputs['releases']==freeze['training_releases'] and training['training']==freeze['training'],'Prior immutable training evidence changed') + launches=outer['native_launches'];require(Counter((item['seed'],item['method']) for item in launches)==Counter({(seed,method):(5 if method=='sgs' else 6) for seed in range(900,904) for method in ('jacobi','sgs','learned')}),'Wrong complete evaluation launch coverage') + policy=outer['native_policy'];expected_maps={'sealed-native','sealed-loader'}|{item['path'] for item in policy['runtime_files'] if item['path']!=policy['loader']['path']} + for launch in launches: + require(set(launch['mappings'])==expected_maps,'Native loaded map closure differs') + require((launch['ready']['n'],launch['ready']['method'],launch['ready']['seed'])==(32,launch['method'],launch['seed']),'Native READY identity differs') + benchmark_result=benchmark(result,freeze) + require([case['seed'] for case in result['accuracy']]==list(range(900,904)),'Missing accuracy trajectory') + checks=[];maxima=[] + for seed in range(900,904): + originals={} + for method in ('jacobi','learned'): + with np.load(directory/f'{method}-{seed}.npz',allow_pickle=False) as loaded: + require(set(loaded.files)=={'pressure','tracer','velocity_0','velocity_1','velocity_2'},'Original NPZ inventory changed') + originals[method]={name:loaded[name] for name in loaded.files} + require(all(value.shape[0]==120 for value in originals[method].values()),'NPZ missing frames') + expected=result['accuracy'][seed-900];require(expected['frames']==120 and len(expected['errors'])==120,'Missing original accuracy errors') + viewport=json.loads((directory/'viewport.json').read_text()) if seed==900 else None + if viewport:require(viewport['n']==32 and viewport['seed']==900 and viewport['dt']==1/60 and viewport['accuracy']==expected,'Original viewport identity/accuracy mismatch') + errors=[];divergences=[] + teacher=trajectory(directory,'jacobi',seed,originals['jacobi'],checks,identity) + predicted=trajectory(directory,'learned',seed,originals['learned'],checks,identity) + for index,((ref,truth),(ai,pred)) in enumerate(zip(teacher,predicted,strict=True)): + error=compare(truth,pred);errors.append(error);divergences.append(ai['actual_divergence_max']) + for key,value in error.items(): + require(math.isfinite(value) and math.isclose(value,expected['errors'][index][key],rel_tol=1e-12,abs_tol=1e-15),'Independent all-field error differs: '+key) + if viewport: + for label,report,arrays in [('reference',ref,truth),('learned',ai,pred)]: + original=viewport[label][index] + require(all(original['report'][key]==value for key,value in report.items()),'Viewport report differs from wire') + require(np.asarray(original['tracer'],dtype='({ok:true,json:async()=>({releases:[candidate]})});late.setHandler(()=>held.promise); const request=late.click('#external-input-choices');late[change]();held.resolve({data:project.data,inputs:{sample:{max_bytes:100}}});await request;late.render();assert.doesNotMatch(late.root.innerHTML,/data-external-input-kind/); } + const sourceRun='1'.repeat(32),otherRun='2'.repeat(32),artifactChoice={artifact_name:'model.pt',role:'checkpoint',bytes:50,sha256:'e'.repeat(64),status:'receipt_verified'}; + const runCandidates=[{run_id:sourceRun,state:'completed',status:'receipt_verified',reason:null,artifacts:[artifactChoice]}, + {run_id:otherRun,state:'completed',status:'receipt_verified',reason:null,artifacts:[{...artifactChoice,sha256:'f'.repeat(64)}]}, + {run_id:'3'.repeat(32),state:'completed',status:'unavailable',reason:'',artifacts:[]}, + {run_id:'4'.repeat(32),state:'failed',status:'receipt_verified',reason:null,artifacts:[artifactChoice]}]; + const runChoices=helpers.choices(project,[candidate],runCandidates); + assert.equal(runChoices.run_artifact.length,2);assert.equal(runChoices.runs.length,3); + const exactBinding={kind:'run_artifact',run_id:sourceRun,artifact_name:'model.pt',expected_sha256:artifactChoice.sha256}; + assert.deepEqual(structuredClone(helpers.binding('run_artifact',runChoices.run_artifact[0])),exactBinding); + assert.match(helpers.html({sample:exactBinding},{sample:{max_bytes:100}},runChoices),/value="run_artifact" selected/); + assert.match(helpers.html({sample:exactBinding},{sample:{max_bytes:100}},runChoices),/value="model.pt" selected/); + assert.match(helpers.html({sample:exactBinding},{sample:{max_bytes:100}},runChoices),/<missing>/); + assert.match(helpers.html({sample:exactBinding},{sample:{max_bytes:100}},helpers.choices(project,[],[])),/exact binding retained/); + bindingUI.window.TuldokAuth.fetch=async url=>({ok:true,json:async()=>url.endsWith('run-artifact-inputs')?{runs:runCandidates}:{releases:[candidate]}}); + await bindingUI.click('#external-input-choices'); + const runSelector={value:'',getAttribute:()=> 'sample'},runGroup={getAttribute:()=> 'sample'}; + bindingUI.fields['[data-external-input-run]']=[runSelector];bindingUI.fields['[data-external-input-run-group]']=[runGroup];bindingUI.render(); + kind.value='run_artifact';kind.onchange();assert.equal(runGroup.hidden,false);assert.equal(runSelector.required,true); + runSelector.value=sourceRun;runSelector.onchange();select.value='model.pt';select.onchange(); + assert.deepEqual(structuredClone(bindingUI.graph.nodes[0].parameters.inputs.sample),exactBinding); + assert.equal(bindingUI.nodes['#external-preflight'].disabled,true);assert.equal(bindingUI.nodes['#external-reviewed'].checked,false); + await bindingUI.click('#external-save');assert.deepEqual(bindingUI.calls.filter(call=>call.route==='graphs/g').at(-1).body.document.nodes[0].parameters.inputs.sample,exactBinding); + runSelector.value=otherRun;runSelector.onchange();assert.equal(bindingUI.graph.nodes[0].parameters.inputs.sample,undefined); + select.value='model.pt';select.onchange();assert.equal(bindingUI.graph.nodes[0].parameters.inputs.sample.expected_sha256,'f'.repeat(64));assert.equal(bindingUI.graph.nodes[0].parameters.inputs.sample.run_id,otherRun); + select.value='../../unlisted';select.onchange();assert.equal(bindingUI.graph.nodes[0].parameters.inputs.sample,undefined); + kind.value='project_reference';kind.onchange();assert.equal(runGroup.hidden,true);assert.equal(runSelector.disabled,true);select.value='source';select.onchange();assert.deepEqual(structuredClone(bindingUI.graph.nodes[0].parameters.inputs.sample),{reference_id:'source',expected_sha256:'b'.repeat(64)}); + bindingUI.window.TuldokAuth.fetch=async url=>{if(url.endsWith('run-artifact-inputs'))throw Error('Candidate service unavailable');return {ok:true,json:async()=>({releases:[candidate]})};}; + const partial=await helpers.catalogs();assert.equal(partial.releases.length,1);assert.equal(partial.runs.length,0);assert.match(partial.errors[0],/Run artifacts/); const progressUI=fixture().ui,completed={id:'r',state:'completed',artifacts:[{name:'model.pt',role:'checkpoint'}]}; const progress={run_id:'r',evidence:'live_unverified',streams:{stdout:{available_bytes:12,start_byte:4,text:'',truncated:true},stderr:null},progress:{format:'tuldok_progress_v1',stage:'loading',completed:1,total:2,metrics:{actual:4}},result:null}; let html=progressUI.progressHTML({...completed,state:'running'},progress);assert.match(html,/Live output · unverified/);assert.match(html,/1 \/ 2/);assert.match(html,/<script>/);assert.doesNotMatch(html,/