Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
- run: build/workspace-training/bin/python -m pip install -r requirements-spatial-qualification.txt
- run: build/workspace-training/bin/python -m unittest discover -s tests -p test_pointclouds.py
- run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py
- run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_cpu_training.py
- run: python -m unittest discover -s tests
- run: node --check static/auth.js
- run: node tests/test_auth_helpers.cjs
Expand Down
329 changes: 329 additions & 0 deletions adapters/rheon/cpu_train.py

Large diffs are not rendered by default.

84 changes: 84 additions & 0 deletions adapters/rheon/cpu_training_project.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
"""Assemble a frozen-release CPU training project; never fit during assembly."""
import argparse
import os
from pathlib import Path
import shutil
import uuid

if __package__:
from . import cpu_train, cpu_compare, generate
else:
import cpu_train
import cpu_compare
import generate

COLD_FILES = ('access.py','audio_assets.py','audio_classification_export.py','audio_export.py','bulk_import.py',
'canonical_release.py','caption_import.py','dataset_releases.py','image_classification_export.py',
'immutable_assets.py','meshes.py','native_text_import.py','recipe_process.py','scientific_fields.py',
'scientific_release.py','text_classification_export.py','text_corpus_export.py','workbench.py')


def definition(sources, runtime_snapshot_sha256):
cpu_train.require(type(runtime_snapshot_sha256) is str and generate.HASH.fullmatch(runtime_snapshot_sha256),
'A qualified runtime snapshot SHA256 is required.')
native='lab/target/release/rheon-trajectory-parallel'
arguments=[]
for name in cpu_train.RELEASE_NAMES:arguments.extend([{'literal':'--release'},{'input':name}])
for name,token in [('build-receipt',{'input':'build_receipt'}),('native-policy',{'native_policy':native}),
('runtime-snapshot-sha256',{'literal':runtime_snapshot_sha256}),('threads',{'parameter':'threads'}),('output',{'output_dir':True})]:
arguments.extend([{'literal':'--'+name},token])
return {'schema_version':1,'format':'external_recipe_v1','id':'rheon-cpu-training',
'name':'Rheon frozen-field CPU fit','version':1,'runtime':'captured_cpu_python_v1',
'runtime_snapshot_sha256':runtime_snapshot_sha256,
'sources':[{'path':name,'sha256':generate.sha(raw)} for name,raw in sorted(sources.items())],
'native_sources':[native],'entrypoint':'cpu_train.py','arguments':arguments,
'parameters':{'threads':{'type':'integer','min':1,'max':8,'default':8}},
'inputs':{**{name:{'max_bytes':40*1024*1024} for name in cpu_train.RELEASE_NAMES},
'build_receipt':{'max_bytes':1024*1024}},
'outputs':[{'path':name,'role':role,'max_bytes':cap,'required':True} for name,role,cap in cpu_train.OUTPUTS],
'environment':{},'resources':{'wall_seconds':600,'max_output_bytes':64*1024*1024}}


def assemble(lab_root,binary,build_receipt,runtime_snapshot_sha256,destination):
lab=Path(lab_root);adapter=Path(__file__).parent;repository=adapter.parents[1]
native=generate.capture(binary,8*1024*1024)[0];build=generate.capture(build_receipt,1024*1024)[0]
cpu_train.require(generate.sha(native)==cpu_compare.BINARY_SHA256
and generate.sha(build)==cpu_compare.BUILD_RECEIPT_SHA256,'Qualified parallel build required.')
sources={'lab/target/release/rheon-trajectory-parallel':native}
for name,digest in cpu_compare.SOURCES.items():
raw=generate.capture(lab/'scripts'/name,65536)[0]
cpu_train.require(generate.sha(raw)==digest,'Original lab algorithm changed.')
sources['lab/scripts/'+name]=raw
for name in ('cpu_train.py','cpu_compare.py','native_transport.py','generate.py'):
sources[name]=generate.capture(adapter/name,256*1024)[0]
for name in COLD_FILES:sources['cold/'+name]=generate.capture(repository/name,8*1024*1024)[0]
recipe=generate.encode(definition(sources,runtime_snapshot_sha256))
manifest={'schema_version':1,'id':uuid.uuid4().hex,'name':'Rheon frozen-field CPU training',
'data':[{'id':'build_receipt','label':'Parallel build evidence','path':'build-receipt.json','sha256':generate.sha(build)}],
'recipes':[{'id':'train','label':'Fit from six frozen training releases','path':'recipe.json','sha256':generate.sha(recipe)}],
'evidence':[],'requirements':[{'id':'reviewed-training','label':'Bind six reviewed train-only releases containing all12 whole seed trajectories'}]}
files={**sources,'build-receipt.json':build,'recipe.json':recipe,'project.json':generate.encode(manifest)}
destination=Path(destination);destination.mkdir(mode=0o700,exist_ok=False)
try:
for name,raw in files.items():
path=destination/name;path.parent.mkdir(parents=True,exist_ok=True)
with path.open('xb') as handle:
handle.write(raw);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno())
for directory in [*sorted((p for p in destination.rglob('*') if p.is_dir()),reverse=True),destination]:
fd=os.open(directory,os.O_RDONLY|os.O_DIRECTORY)
try:os.fsync(fd)
finally:os.close(fd)
except BaseException:
shutil.rmtree(destination);raise
return manifest


def main():
parser=argparse.ArgumentParser(description=__doc__)
for name in ('lab-root','binary','build-receipt','runtime-snapshot-sha256','destination'):parser.add_argument('--'+name,required=True)
args=parser.parse_args()
manifest=assemble(args.lab_root,args.binary,args.build_receipt,args.runtime_snapshot_sha256,args.destination)
print(generate.encode({'project_id':manifest['id'],'manifest':str(Path(args.destination)/'project.json')}).decode())


if __name__=='__main__':main()
9 changes: 5 additions & 4 deletions adapters/rheon/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
'Rheon': '2209012bc5f902693d0f695ec1c2d854126af317'}
LIBRARY_DIRECTORY = '/usr/lib/x86_64-linux-gnu'
LOADER_ARGUMENTS = ['--inhibit-cache', '--library-path', LIBRARY_DIRECTORY, '--glibc-hwcaps-mask', '']
WIRE_LIMIT = 4 * 1024 * 1024
WIRE_LIMIT = 12 * 1024 * 1024
DIAGNOSTIC_LIMIT = 65536
HASH = re.compile(r'^[a-f0-9]{64}$')

Expand Down Expand Up @@ -54,9 +54,9 @@ def closed(value, keys):


def controls(grid, frames, seed):
for value, low, high in ((grid, 8, 32), (frames, 1, 2), (seed, 0, 2147483647)):
for value, low, high in ((grid, 8, 32), (frames, 1, 8), (seed, 0, 2147483647)):
if type(value) is not int or not low <= value <= high:
raise ValueError('Supported controls: grid 8–32, frames 1–2, seed 0–2147483647; Jacobi only.')
raise ValueError('Supported controls: grid 8–32, frames 1–8, seed 0–2147483647; Jacobi only.')
return {'grid': grid, 'frames': frames, 'seed': seed, 'method': 'jacobi'}


Expand Down Expand Up @@ -320,7 +320,8 @@ def generate(native_path, build_path, policy_path, grid, frames, seed, output):
receipt_raw = encode(receipt)
bundle = scientific_bundle(wire, request, build_raw, receipt_raw, policy_raw)
outputs = {'fields.zip': bundle, 'wire.bin': wire, 'producer-receipt.json': receipt_raw, 'native-diagnostics.txt': diagnostics}
if sum(map(len, outputs.values())) > 16 * 1024 * 1024: raise ValueError('Output budget exceeded.')
if len(bundle) > 20 * 1024 * 1024: raise ValueError('Scientific bundle budget exceeded.')
if sum(map(len, outputs.values())) > 40 * 1024 * 1024: raise ValueError('Output budget exceeded.')
for name, data in outputs.items():
with (output/name).open('xb') as handle: handle.write(data); handle.flush(); os.fsync(handle.fileno())
print(json.dumps({'stage': 'generated', 'frames': frames, 'fields': 6, 'bundle_bytes': len(bundle)}), flush=True)
Expand Down
41 changes: 39 additions & 2 deletions adapters/rheon/native_transport.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
from pathlib import Path
import struct
import subprocess
import tempfile
import sys

if __package__:
from . import generate
Expand Down Expand Up @@ -40,7 +42,14 @@ def exact(stream, size):
class Transport:
PIPE = subprocess.PIPE

def __init__(self, binary, policy_raw):
def __init__(self, binary, policy_raw, *, allowed_seeds=(900, 901, 902, 903), capture_diagnostics=False):
if type(allowed_seeds) is not tuple or allowed_seeds not in ((900, 901, 902, 903), (500, 501)):
raise ValueError('Unsupported reviewed native seed scope.')
if type(capture_diagnostics) is not bool:
raise ValueError('Invalid diagnostic capture flag.')
self.capture_diagnostics = capture_diagnostics
self.diagnostics = []
self.allowed_seeds = tuple(str(seed) for seed in allowed_seeds)
self.binary = str(Path(binary))
self.native, _ = generate.capture(binary, 8 * 1024 * 1024)
self.policy, self.observed = generate.native_policy(policy_raw, self.native)
Expand All @@ -49,19 +58,45 @@ def __init__(self, binary, policy_raw):
def Popen(self, args, **kwargs):
if (type(args) is not list or len(args) != 4 or args[0] != self.binary
or args[1] != '32' or args[2] not in ('jacobi', 'sgs', 'learned')
or args[3] not in ('900', '901', '902', '903')
or args[3] not in getattr(self, 'allowed_seeds', ('900', '901', '902', '903'))
or kwargs != {'stdin': subprocess.PIPE, 'stdout': subprocess.PIPE}):
raise ValueError('Native launch differs from admitted CPU comparison.')
if os.path.lexists('/etc/ld.so.preload'):
raise ValueError('System preload is unsupported.')
native_fd = generate.seal('rheon-cpu-native', self.native)
loader_fd = None
process = None
diagnostic = None
try:
loader_fd = generate.seal('rheon-cpu-loader', self.observed[self.policy['loader']['path']][0])
command = [f'/proc/self/fd/{loader_fd}', *self.policy['loader_arguments'],
f'/proc/self/fd/{native_fd}', *args[1:]]
if self.capture_diagnostics:
diagnostic = tempfile.TemporaryFile()
kwargs = dict(kwargs, stderr=diagnostic)
process = subprocess.Popen(command, pass_fds=(native_fd, loader_fd), **kwargs)
if diagnostic is not None:
original_wait = process.wait
stream = diagnostic
captured = False
def wait_with_diagnostics(*wait_args, **wait_kwargs):
nonlocal captured
code = original_wait(*wait_args, **wait_kwargs)
if not captured:
captured = True
stream.seek(0)
raw = stream.read(65537)
stream.close()
if len(raw) > 65536:
raise ValueError('Native diagnostics exceed bound.')
self.diagnostics.append({'seed': int(args[3]), 'method': args[2],
'exit_code': code, 'stderr_hex': raw.hex(),
'stderr_sha256': generate.sha(raw)})
if raw:
sys.stderr.buffer.write(raw)
sys.stderr.buffer.flush()
return code
process.wait = wait_with_diagnostics
header = exact(process.stdout, 4)
length, = struct.unpack('<I', header)
if length > 65536:
Expand All @@ -88,6 +123,8 @@ def Popen(self, args, **kwargs):
process.stdout.close()
raise
finally:
if diagnostic is not None and process is None:
diagnostic.close()
os.close(native_fd)
if loader_fd is not None:
os.close(loader_fd)
6 changes: 3 additions & 3 deletions adapters/rheon/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@ def definition(wrapper, native):
{'literal': '--grid'}, {'parameter': 'grid'}, {'literal': '--frames'}, {'parameter': 'frames'},
{'literal': '--seed'}, {'parameter': 'seed'}, {'literal': '--output'}, {'output_dir': True}],
'parameters': {'grid': {'type': 'integer', 'min': 8, 'max': 32, 'default': 32},
'frames': {'type': 'integer', 'min': 1, 'max': 2, 'default': 2},
'frames': {'type': 'integer', 'min': 1, 'max': 8, 'default': 2},
'seed': {'type': 'integer', 'min': 0, 'max': 2147483647, 'default': 100}},
'inputs': {'build_receipt': {'max_bytes': generate.DIAGNOSTIC_LIMIT}},
'outputs': [{'path': name, 'role': role, 'max_bytes': limit, 'required': True} for name, role, limit in (
('fields.zip', 'scientific_fields', 8*1024*1024), ('wire.bin', 'native_wire', generate.WIRE_LIMIT),
('fields.zip', 'scientific_fields', 20*1024*1024), ('wire.bin', 'native_wire', generate.WIRE_LIMIT),
('producer-receipt.json', 'producer_receipt', generate.DIAGNOSTIC_LIMIT),
('native-diagnostics.txt', 'native_diagnostics', generate.DIAGNOSTIC_LIMIT))],
'environment': {}, 'resources': {'wall_seconds': 120, 'max_output_bytes': 16*1024*1024}}
'environment': {}, 'resources': {'wall_seconds': 120, 'max_output_bytes': 40*1024*1024}}


def assemble(binary, receipt, destination):
Expand Down
4 changes: 4 additions & 0 deletions app.py
Original file line number Diff line number Diff line change
Expand Up @@ -582,13 +582,17 @@ def _get(self):
return self.reply(dataset.runs.list())
if len(parts) == 2 and parts[0] == 'runs':
return self.reply(dataset.runs.get(parts[1]))
if len(parts) == 3 and parts[0] == 'runs' and parts[2] == 'progress':
return self.reply(dataset.runs.external_progress(parts[1]))
if len(parts) == 3 and parts[0] == 'runs' and parts[2] == 'predictions':
query = parse_qs(urlsplit(self.path).query)
return self.reply(dataset.runs.predictions(parts[1], query.get('split', ['validation'])[-1]))
if len(parts) == 4 and parts[0] == 'runs' and parts[2] == 'artifacts':
data, media_type = dataset.runs.artifact(parts[1], parts[3])
return self.reply(data, content_type=media_type)
return self.reply({'error': 'Workspace route not found.', 'code': 'not_found'}, 404)
if path == '/api/workbench/releases/input-candidates':
return self.reply(dataset.releases.input_candidates())
if path == '/api/workbench/selections':
return self.reply(dataset.selections.list())
if path == '/api/workbench/searches':
Expand Down
2 changes: 1 addition & 1 deletion bulk_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import hashlib
import json
import re
import psycopg

from workbench import WorkbenchError, text_value

Expand Down Expand Up @@ -86,6 +85,7 @@ def find_result(workbench, request_id):


def import_row(workbench, body):
import psycopg
workbench.access.require('edit', 'bulk_import.import_row')
if set(body) - {'source_name', 'row_number', 'line', 'request_id', 'image_name', 'image'}:
raise WorkbenchError('Unsupported row admission fields.')
Expand Down
4 changes: 2 additions & 2 deletions caption_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
import json
import re
import secrets
import psycopg

from PIL import Image

from bulk_import import find_result, invalid_constant, leaf_name, result, unique_object
from dataset_releases import CAPTION_CONSUMER, CAPTION_FORMAT, CAPTION_SPLITS, SPLITS, connected_components
Expand Down Expand Up @@ -260,6 +258,8 @@ def prepare(self, body):

@authorized('edit', 'caption_import.admit')
def admit(self, body):
from PIL import Image
import psycopg
if set(body) != {'token', 'asset', 'image', 'request_id'}:
invalid('Unsupported caption admission fields.')
if not isinstance(body['request_id'], str) or not IDENTIFIER.fullmatch(body['request_id']):
Expand Down
Loading
Loading